2026-02-19T20:24:52.260 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-19T20:24:52.279 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-19T20:24:52.360 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645 branch: wip-yuri-testing-2026-02-18-1820 description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '59645' kernel: branch: distro kdb: true sha1: distro ktype: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial no_nested_subset: false os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: wip-yuri-testing-2026-02-18-1820 ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: simple client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluefs allocator: hybrid bluestore allocator: hybrid bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: 802bf111cd55e57f5842153d7ba2b8b964aa818a ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 802bf111cd55e57f5842153d7ba2b8b964aa818a thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-yuri-testing-2026-02-18-1820 sha1: 802bf111cd55e57f5842153d7ba2b8b964aa818a owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 9494 sha1: 802bf111cd55e57f5842153d7ba2b8b964aa818a sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-yuri-testing-2026-02-18-1820 suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_802bf111cd55e57f5842153d7ba2b8b964aa818a/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: 802bf111cd55e57f5842153d7ba2b8b964aa818a targets: trial013.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAjNtrYticDCX9pnZhRVVnflbZnQIugXBMd8zIps+2p71TYf4S3fwnv57bFRq5XnJ9JYrboYaeEBqKxKhQY5nNI= trial151.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIjmP8lkx+u8iIeShbMgEV9ajpbAMXWwo0tojC0hZTyrhgo4Ro+8UWsRhCu8C++sBWAZLI6gwDwoff55PAEivAM= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: pool RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: journal RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-19_15:47:56 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699 2026-02-19T20:24:52.360 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_802bf111cd55e57f5842153d7ba2b8b964aa818a/qa; will attempt to use it 2026-02-19T20:24:52.360 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_802bf111cd55e57f5842153d7ba2b8b964aa818a/qa/tasks 2026-02-19T20:24:52.360 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-19T20:24:52.362 INFO:teuthology.task.internal:Checking packages... 2026-02-19T20:24:52.362 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '802bf111cd55e57f5842153d7ba2b8b964aa818a' 2026-02-19T20:24:52.362 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-19T20:24:52.362 INFO:teuthology.packaging:ref: None 2026-02-19T20:24:52.362 INFO:teuthology.packaging:tag: None 2026-02-19T20:24:52.362 INFO:teuthology.packaging:branch: wip-yuri-testing-2026-02-18-1820 2026-02-19T20:24:52.362 INFO:teuthology.packaging:sha1: 802bf111cd55e57f5842153d7ba2b8b964aa818a 2026-02-19T20:24:52.362 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=wip-yuri-testing-2026-02-18-1820 2026-02-19T20:24:52.492 INFO:teuthology.task.internal:Found packages for ceph version 20.3.0-5336-g802bf111-1jammy 2026-02-19T20:24:52.493 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-19T20:24:52.505 INFO:teuthology.task.internal:no buildpackages task found 2026-02-19T20:24:52.505 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-19T20:24:52.519 INFO:teuthology.task.internal:Saving configuration 2026-02-19T20:24:52.535 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-19T20:24:52.547 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-19T20:24:52.624 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial013.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-19 20:21:14.429581', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK/ULTiqTwoiz/T8BJszTir2n2JHF5kU/yLc0JtEDcbQ'} 2026-02-19T20:24:52.680 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial151.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-19 20:21:14.430201', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINjlMHn00aVVMjiUCPGK2fqjl5iwz9EhjcACSrqorAcf'} 2026-02-19T20:24:52.680 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-19T20:24:52.691 INFO:teuthology.task.internal:roles: ubuntu@trial013.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-19T20:24:52.692 INFO:teuthology.task.internal:roles: ubuntu@trial151.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-19T20:24:52.692 INFO:teuthology.run_tasks:Running task console_log... 2026-02-19T20:24:52.825 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7302571d1620>, signals=[15]) 2026-02-19T20:24:52.825 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-19T20:24:52.836 INFO:teuthology.task.internal:Opening connections... 2026-02-19T20:24:52.836 DEBUG:teuthology.task.internal:connecting to ubuntu@trial013.front.sepia.ceph.com 2026-02-19T20:24:52.837 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial013.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-19T20:24:52.899 DEBUG:teuthology.task.internal:connecting to ubuntu@trial151.front.sepia.ceph.com 2026-02-19T20:24:52.899 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-19T20:24:52.960 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-19T20:24:52.972 DEBUG:teuthology.orchestra.run.trial013:> uname -m 2026-02-19T20:24:52.976 INFO:teuthology.orchestra.run.trial013.stdout:x86_64 2026-02-19T20:24:52.976 DEBUG:teuthology.orchestra.run.trial013:> cat /etc/os-release 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:NAME="Ubuntu" 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:VERSION_ID="22.04" 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:VERSION_CODENAME=jammy 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:ID=ubuntu 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:ID_LIKE=debian 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:HOME_URL="https://www.ubuntu.com/" 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-02-19T20:24:53.020 INFO:teuthology.orchestra.run.trial013.stdout:UBUNTU_CODENAME=jammy 2026-02-19T20:24:53.021 INFO:teuthology.lock.ops:Updating trial013.front.sepia.ceph.com on lock server 2026-02-19T20:24:53.086 DEBUG:teuthology.orchestra.run.trial151:> uname -m 2026-02-19T20:24:53.089 INFO:teuthology.orchestra.run.trial151.stdout:x86_64 2026-02-19T20:24:53.089 DEBUG:teuthology.orchestra.run.trial151:> cat /etc/os-release 2026-02-19T20:24:53.132 INFO:teuthology.orchestra.run.trial151.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-02-19T20:24:53.132 INFO:teuthology.orchestra.run.trial151.stdout:NAME="Ubuntu" 2026-02-19T20:24:53.132 INFO:teuthology.orchestra.run.trial151.stdout:VERSION_ID="22.04" 2026-02-19T20:24:53.132 INFO:teuthology.orchestra.run.trial151.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-02-19T20:24:53.132 INFO:teuthology.orchestra.run.trial151.stdout:VERSION_CODENAME=jammy 2026-02-19T20:24:53.132 INFO:teuthology.orchestra.run.trial151.stdout:ID=ubuntu 2026-02-19T20:24:53.132 INFO:teuthology.orchestra.run.trial151.stdout:ID_LIKE=debian 2026-02-19T20:24:53.132 INFO:teuthology.orchestra.run.trial151.stdout:HOME_URL="https://www.ubuntu.com/" 2026-02-19T20:24:53.133 INFO:teuthology.orchestra.run.trial151.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-02-19T20:24:53.133 INFO:teuthology.orchestra.run.trial151.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-02-19T20:24:53.133 INFO:teuthology.orchestra.run.trial151.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-02-19T20:24:53.133 INFO:teuthology.orchestra.run.trial151.stdout:UBUNTU_CODENAME=jammy 2026-02-19T20:24:53.133 INFO:teuthology.lock.ops:Updating trial151.front.sepia.ceph.com on lock server 2026-02-19T20:24:53.202 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-19T20:24:53.214 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-19T20:24:53.225 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-19T20:24:53.225 DEBUG:teuthology.orchestra.run.trial013:> test '!' -e /home/ubuntu/cephtest 2026-02-19T20:24:53.226 DEBUG:teuthology.orchestra.run.trial151:> test '!' -e /home/ubuntu/cephtest 2026-02-19T20:24:53.229 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-19T20:24:53.240 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-19T20:24:53.240 DEBUG:teuthology.orchestra.run.trial013:> test -z $(ls -A /var/lib/ceph) 2026-02-19T20:24:53.270 DEBUG:teuthology.orchestra.run.trial151:> test -z $(ls -A /var/lib/ceph) 2026-02-19T20:24:53.273 INFO:teuthology.orchestra.run.trial013.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-02-19T20:24:53.274 INFO:teuthology.orchestra.run.trial151.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-02-19T20:24:53.274 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-19T20:24:53.403 INFO:teuthology.run_tasks:Running task kernel... 2026-02-19T20:24:53.422 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-19T20:24:53.423 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-19T20:24:53.423 DEBUG:teuthology.orchestra.run.trial013:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-19T20:24:53.423 DEBUG:teuthology.orchestra.run.trial151:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-19T20:24:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T20:24:53.426 DEBUG:teuthology.orchestra.run.trial013:> uname -r 2026-02-19T20:24:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T20:24:53.426 DEBUG:teuthology.orchestra.run.trial151:> uname -r 2026-02-19T20:24:53.476 INFO:teuthology.orchestra.run.trial013.stdout:5.15.0-170-generic 2026-02-19T20:24:53.476 INFO:teuthology.task.kernel:Running kernel on trial013: 5.15.0-170-generic 2026-02-19T20:24:53.476 DEBUG:teuthology.orchestra.run.trial013:> sudo apt-get clean 2026-02-19T20:24:53.477 INFO:teuthology.orchestra.run.trial151.stdout:5.15.0-170-generic 2026-02-19T20:24:53.477 INFO:teuthology.task.kernel:Running kernel on trial151: 5.15.0-170-generic 2026-02-19T20:24:53.477 DEBUG:teuthology.orchestra.run.trial151:> sudo apt-get clean 2026-02-19T20:24:53.534 DEBUG:teuthology.orchestra.run.trial013:> sudo apt-get update 2026-02-19T20:24:53.535 DEBUG:teuthology.orchestra.run.trial151:> sudo apt-get update 2026-02-19T20:24:53.718 INFO:teuthology.orchestra.run.trial151.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-19T20:24:53.721 INFO:teuthology.orchestra.run.trial013.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-19T20:24:53.726 INFO:teuthology.orchestra.run.trial151.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2026-02-19T20:24:53.728 INFO:teuthology.orchestra.run.trial013.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2026-02-19T20:24:53.764 INFO:teuthology.orchestra.run.trial151.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2026-02-19T20:24:53.767 INFO:teuthology.orchestra.run.trial013.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2026-02-19T20:24:53.779 INFO:teuthology.orchestra.run.trial151.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2026-02-19T20:24:53.783 INFO:teuthology.orchestra.run.trial013.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2026-02-19T20:24:53.862 INFO:teuthology.orchestra.run.trial151.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [3245 kB] 2026-02-19T20:24:53.863 INFO:teuthology.orchestra.run.trial013.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [3245 kB] 2026-02-19T20:24:53.927 INFO:teuthology.orchestra.run.trial151.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [495 kB] 2026-02-19T20:24:53.929 INFO:teuthology.orchestra.run.trial013.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [495 kB] 2026-02-19T20:24:53.930 INFO:teuthology.orchestra.run.trial151.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [19.1 kB] 2026-02-19T20:24:53.931 INFO:teuthology.orchestra.run.trial151.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1256 kB] 2026-02-19T20:24:53.932 INFO:teuthology.orchestra.run.trial013.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [19.1 kB] 2026-02-19T20:24:53.932 INFO:teuthology.orchestra.run.trial013.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1256 kB] 2026-02-19T20:24:53.939 INFO:teuthology.orchestra.run.trial151.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [314 kB] 2026-02-19T20:24:53.940 INFO:teuthology.orchestra.run.trial013.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [314 kB] 2026-02-19T20:24:53.941 INFO:teuthology.orchestra.run.trial151.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [30.3 kB] 2026-02-19T20:24:53.941 INFO:teuthology.orchestra.run.trial151.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [5310 kB] 2026-02-19T20:24:53.942 INFO:teuthology.orchestra.run.trial013.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [30.3 kB] 2026-02-19T20:24:53.942 INFO:teuthology.orchestra.run.trial013.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [5310 kB] 2026-02-19T20:24:53.969 INFO:teuthology.orchestra.run.trial151.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [1005 kB] 2026-02-19T20:24:53.970 INFO:teuthology.orchestra.run.trial013.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [1005 kB] 2026-02-19T20:24:53.972 INFO:teuthology.orchestra.run.trial151.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2983 kB] 2026-02-19T20:24:53.973 INFO:teuthology.orchestra.run.trial013.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2983 kB] 2026-02-19T20:24:53.981 INFO:teuthology.orchestra.run.trial151.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-security/main Translation-en [427 kB] 2026-02-19T20:24:53.982 INFO:teuthology.orchestra.run.trial013.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-security/main Translation-en [427 kB] 2026-02-19T20:24:53.983 INFO:teuthology.orchestra.run.trial151.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [14.1 kB] 2026-02-19T20:24:53.983 INFO:teuthology.orchestra.run.trial151.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1018 kB] 2026-02-19T20:24:53.983 INFO:teuthology.orchestra.run.trial013.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [14.1 kB] 2026-02-19T20:24:53.983 INFO:teuthology.orchestra.run.trial013.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1018 kB] 2026-02-19T20:24:53.987 INFO:teuthology.orchestra.run.trial151.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [22.7 kB] 2026-02-19T20:24:53.987 INFO:teuthology.orchestra.run.trial151.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [5147 kB] 2026-02-19T20:24:53.988 INFO:teuthology.orchestra.run.trial013.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [22.7 kB] 2026-02-19T20:24:53.988 INFO:teuthology.orchestra.run.trial013.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [5147 kB] 2026-02-19T20:24:54.004 INFO:teuthology.orchestra.run.trial151.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy-security/restricted Translation-en [979 kB] 2026-02-19T20:24:54.005 INFO:teuthology.orchestra.run.trial013.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy-security/restricted Translation-en [979 kB] 2026-02-19T20:24:57.834 INFO:teuthology.orchestra.run.trial151.stdout:Fetched 22.6 MB in 2s (13.7 MB/s) 2026-02-19T20:24:57.953 INFO:teuthology.orchestra.run.trial013.stdout:Fetched 22.6 MB in 2s (13.3 MB/s) 2026-02-19T20:24:58.356 INFO:teuthology.orchestra.run.trial151.stdout:Reading package lists... 2026-02-19T20:24:58.369 DEBUG:teuthology.orchestra.run.trial151:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-19T20:24:58.402 INFO:teuthology.orchestra.run.trial151.stdout:Reading package lists... 2026-02-19T20:24:58.488 INFO:teuthology.orchestra.run.trial013.stdout:Reading package lists... 2026-02-19T20:24:58.501 DEBUG:teuthology.orchestra.run.trial013:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-19T20:24:58.508 INFO:teuthology.orchestra.run.trial151.stdout:Building dependency tree... 2026-02-19T20:24:58.509 INFO:teuthology.orchestra.run.trial151.stdout:Reading state information... 2026-02-19T20:24:58.534 INFO:teuthology.orchestra.run.trial013.stdout:Reading package lists... 2026-02-19T20:24:58.591 INFO:teuthology.orchestra.run.trial151.stdout:linux-image-generic is already the newest version (5.15.0.170.159). 2026-02-19T20:24:58.591 INFO:teuthology.orchestra.run.trial151.stdout:linux-image-generic set to manually installed. 2026-02-19T20:24:58.592 INFO:teuthology.orchestra.run.trial151.stdout:The following packages were automatically installed and are no longer required: 2026-02-19T20:24:58.592 INFO:teuthology.orchestra.run.trial151.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-19T20:24:58.592 INFO:teuthology.orchestra.run.trial151.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-19T20:24:58.592 INFO:teuthology.orchestra.run.trial151.stdout: sg3-utils-udev 2026-02-19T20:24:58.592 INFO:teuthology.orchestra.run.trial151.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-19T20:24:58.610 INFO:teuthology.orchestra.run.trial151.stdout:0 upgraded, 0 newly installed, 0 to remove and 13 not upgraded. 2026-02-19T20:24:58.612 DEBUG:teuthology.orchestra.run.trial151:> dpkg -s linux-image-generic 2026-02-19T20:24:58.648 INFO:teuthology.orchestra.run.trial013.stdout:Building dependency tree... 2026-02-19T20:24:58.649 INFO:teuthology.orchestra.run.trial013.stdout:Reading state information... 2026-02-19T20:24:58.663 INFO:teuthology.orchestra.run.trial151.stdout:Package: linux-image-generic 2026-02-19T20:24:58.664 INFO:teuthology.orchestra.run.trial151.stdout:Status: install ok installed 2026-02-19T20:24:58.664 INFO:teuthology.orchestra.run.trial151.stdout:Priority: optional 2026-02-19T20:24:58.664 INFO:teuthology.orchestra.run.trial151.stdout:Section: kernel 2026-02-19T20:24:58.664 INFO:teuthology.orchestra.run.trial151.stdout:Installed-Size: 21 2026-02-19T20:24:58.664 INFO:teuthology.orchestra.run.trial151.stdout:Maintainer: Ubuntu Kernel Team 2026-02-19T20:24:58.664 INFO:teuthology.orchestra.run.trial151.stdout:Architecture: amd64 2026-02-19T20:24:58.664 INFO:teuthology.orchestra.run.trial151.stdout:Source: linux-meta 2026-02-19T20:24:58.664 INFO:teuthology.orchestra.run.trial151.stdout:Version: 5.15.0.170.159 2026-02-19T20:24:58.664 INFO:teuthology.orchestra.run.trial151.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-19T20:24:58.664 INFO:teuthology.orchestra.run.trial151.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-19T20:24:58.665 INFO:teuthology.orchestra.run.trial151.stdout:Recommends: thermald 2026-02-19T20:24:58.665 INFO:teuthology.orchestra.run.trial151.stdout:Description: Generic Linux kernel image 2026-02-19T20:24:58.665 INFO:teuthology.orchestra.run.trial151.stdout: This package will always depend on the latest generic kernel image 2026-02-19T20:24:58.665 INFO:teuthology.orchestra.run.trial151.stdout: available. 2026-02-19T20:24:58.665 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-19T20:24:58.665 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-19T20:24:58.666 DEBUG:teuthology.orchestra.run.trial151:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-19T20:24:58.713 INFO:teuthology.orchestra.run.trial151.stdout:ttyS1 2026-02-19T20:24:58.738 INFO:teuthology.orchestra.run.trial013.stdout:linux-image-generic is already the newest version (5.15.0.170.159). 2026-02-19T20:24:58.738 INFO:teuthology.orchestra.run.trial013.stdout:linux-image-generic set to manually installed. 2026-02-19T20:24:58.738 INFO:teuthology.orchestra.run.trial013.stdout:The following packages were automatically installed and are no longer required: 2026-02-19T20:24:58.738 INFO:teuthology.orchestra.run.trial013.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-19T20:24:58.738 INFO:teuthology.orchestra.run.trial013.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-19T20:24:58.738 INFO:teuthology.orchestra.run.trial013.stdout: sg3-utils-udev 2026-02-19T20:24:58.739 INFO:teuthology.orchestra.run.trial013.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-19T20:24:58.740 DEBUG:teuthology.parallel:result is None 2026-02-19T20:24:58.758 INFO:teuthology.orchestra.run.trial013.stdout:0 upgraded, 0 newly installed, 0 to remove and 13 not upgraded. 2026-02-19T20:24:58.760 DEBUG:teuthology.orchestra.run.trial013:> dpkg -s linux-image-generic 2026-02-19T20:24:58.812 INFO:teuthology.orchestra.run.trial013.stdout:Package: linux-image-generic 2026-02-19T20:24:58.812 INFO:teuthology.orchestra.run.trial013.stdout:Status: install ok installed 2026-02-19T20:24:58.812 INFO:teuthology.orchestra.run.trial013.stdout:Priority: optional 2026-02-19T20:24:58.812 INFO:teuthology.orchestra.run.trial013.stdout:Section: kernel 2026-02-19T20:24:58.812 INFO:teuthology.orchestra.run.trial013.stdout:Installed-Size: 21 2026-02-19T20:24:58.812 INFO:teuthology.orchestra.run.trial013.stdout:Maintainer: Ubuntu Kernel Team 2026-02-19T20:24:58.812 INFO:teuthology.orchestra.run.trial013.stdout:Architecture: amd64 2026-02-19T20:24:58.813 INFO:teuthology.orchestra.run.trial013.stdout:Source: linux-meta 2026-02-19T20:24:58.813 INFO:teuthology.orchestra.run.trial013.stdout:Version: 5.15.0.170.159 2026-02-19T20:24:58.813 INFO:teuthology.orchestra.run.trial013.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-19T20:24:58.813 INFO:teuthology.orchestra.run.trial013.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-19T20:24:58.813 INFO:teuthology.orchestra.run.trial013.stdout:Recommends: thermald 2026-02-19T20:24:58.813 INFO:teuthology.orchestra.run.trial013.stdout:Description: Generic Linux kernel image 2026-02-19T20:24:58.813 INFO:teuthology.orchestra.run.trial013.stdout: This package will always depend on the latest generic kernel image 2026-02-19T20:24:58.813 INFO:teuthology.orchestra.run.trial013.stdout: available. 2026-02-19T20:24:58.813 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-19T20:24:58.814 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-19T20:24:58.814 DEBUG:teuthology.orchestra.run.trial013:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-19T20:24:58.861 INFO:teuthology.orchestra.run.trial013.stdout:ttyS1 2026-02-19T20:24:58.879 DEBUG:teuthology.parallel:result is None 2026-02-19T20:24:58.879 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-19T20:24:58.890 INFO:teuthology.task.internal:Creating test directory... 2026-02-19T20:24:58.890 DEBUG:teuthology.orchestra.run.trial013:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-19T20:24:58.922 DEBUG:teuthology.orchestra.run.trial151:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-19T20:24:58.927 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-19T20:24:58.938 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-19T20:24:58.951 INFO:teuthology.task.internal:Creating archive directory... 2026-02-19T20:24:58.951 DEBUG:teuthology.orchestra.run.trial013:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-19T20:24:58.971 DEBUG:teuthology.orchestra.run.trial151:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-19T20:24:58.986 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-19T20:24:58.998 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-19T20:24:58.998 DEBUG:teuthology.orchestra.run.trial013:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-19T20:24:59.019 DEBUG:teuthology.orchestra.run.trial151:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-19T20:24:59.026 INFO:teuthology.orchestra.run.trial013.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-19T20:24:59.030 INFO:teuthology.orchestra.run.trial151.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-19T20:24:59.032 INFO:teuthology.orchestra.run.trial013.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-19T20:24:59.036 INFO:teuthology.orchestra.run.trial151.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-19T20:24:59.038 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-19T20:24:59.050 INFO:teuthology.task.internal:Configuring sudo... 2026-02-19T20:24:59.050 DEBUG:teuthology.orchestra.run.trial013:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-19T20:24:59.079 DEBUG:teuthology.orchestra.run.trial151:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-19T20:24:59.093 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-19T20:24:59.116 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-19T20:24:59.116 DEBUG:teuthology.orchestra.run.trial013:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-19T20:24:59.131 DEBUG:teuthology.orchestra.run.trial151:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-19T20:24:59.137 DEBUG:teuthology.orchestra.run.trial013:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-19T20:24:59.177 DEBUG:teuthology.orchestra.run.trial013:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-19T20:24:59.221 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:24:59.221 DEBUG:teuthology.orchestra.run.trial013:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-19T20:24:59.268 DEBUG:teuthology.orchestra.run.trial151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-19T20:24:59.272 DEBUG:teuthology.orchestra.run.trial151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-19T20:24:59.316 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:24:59.317 DEBUG:teuthology.orchestra.run.trial151:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-19T20:24:59.366 DEBUG:teuthology.orchestra.run.trial013:> sudo service rsyslog restart 2026-02-19T20:24:59.367 DEBUG:teuthology.orchestra.run.trial151:> sudo service rsyslog restart 2026-02-19T20:24:59.426 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-19T20:24:59.445 INFO:teuthology.task.internal:Starting timer... 2026-02-19T20:24:59.445 INFO:teuthology.run_tasks:Running task pcp... 2026-02-19T20:24:59.460 INFO:teuthology.run_tasks:Running task selinux... 2026-02-19T20:24:59.472 DEBUG:teuthology.task.selinux:Excluding trial013: OS 'ubuntu' does not support SELinux 2026-02-19T20:24:59.472 DEBUG:teuthology.task.selinux:Excluding trial151: OS 'ubuntu' does not support SELinux 2026-02-19T20:24:59.472 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-19T20:24:59.472 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2026-02-19T20:24:59.472 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-19T20:24:59.472 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-19T20:24:59.483 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-19T20:24:59.491 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-19T20:24:59.492 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial013.front.sepia.ceph.com,trial151.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-19T20:27:25.344 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial013.front.sepia.ceph.com'), Remote(name='ubuntu@trial151.front.sepia.ceph.com')] 2026-02-19T20:27:25.347 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial013.front.sepia.ceph.com' 2026-02-19T20:27:25.350 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial013.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-19T20:27:25.459 DEBUG:teuthology.orchestra.run.trial013:> true 2026-02-19T20:27:25.708 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial013.front.sepia.ceph.com' 2026-02-19T20:27:25.709 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial151.front.sepia.ceph.com' 2026-02-19T20:27:25.710 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-19T20:27:25.813 DEBUG:teuthology.orchestra.run.trial151:> true 2026-02-19T20:27:26.063 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial151.front.sepia.ceph.com' 2026-02-19T20:27:26.064 INFO:teuthology.run_tasks:Running task clock... 2026-02-19T20:27:26.085 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-19T20:27:26.085 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-19T20:27:26.086 DEBUG:teuthology.orchestra.run.trial013:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-19T20:27:26.089 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-19T20:27:26.089 DEBUG:teuthology.orchestra.run.trial151:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-19T20:27:26.107 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-02-19T20:27:26.108 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: Command line: ntpd -gq 2026-02-19T20:27:26.108 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: ---------------------------------------------------- 2026-02-19T20:27:26.108 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: ntp-4 is maintained by Network Time Foundation, 2026-02-19T20:27:26.108 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-02-19T20:27:26.108 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: corporation. Support and training for ntp-4 are 2026-02-19T20:27:26.108 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: available at https://www.nwtime.org/support 2026-02-19T20:27:26.108 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: ---------------------------------------------------- 2026-02-19T20:27:26.108 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: proto: precision = 0.030 usec (-25) 2026-02-19T20:27:26.108 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: basedate set to 2022-02-04 2026-02-19T20:27:26.108 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: gps base set to 2022-02-06 (week 2196) 2026-02-19T20:27:26.109 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-19T20:27:26.109 INFO:teuthology.orchestra.run.trial013.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-19T20:27:26.109 INFO:teuthology.orchestra.run.trial013.stderr:restrict ::: KOD does nothing without LIMITED. 2026-02-19T20:27:26.110 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: restrict ::: KOD does nothing without LIMITED. 2026-02-19T20:27:26.110 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: Listen and drop on 0 v6wildcard [::]:123 2026-02-19T20:27:26.110 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-02-19T20:27:26.110 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: Listen normally on 2 lo 127.0.0.1:123 2026-02-19T20:27:26.110 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: Listen normally on 3 enp1s0f0 10.20.193.13:123 2026-02-19T20:27:26.111 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: Listen normally on 4 lo [::1]:123 2026-02-19T20:27:26.111 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: Listen normally on 5 enp1s0f0 [fe80::925a:8ff:fe77:6010%2]:123 2026-02-19T20:27:26.111 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:25 ntpd[9417]: Listening on routing socket on fd #22 for interface updates 2026-02-19T20:27:26.121 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-02-19T20:27:26.121 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: Command line: ntpd -gq 2026-02-19T20:27:26.121 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: ---------------------------------------------------- 2026-02-19T20:27:26.121 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: ntp-4 is maintained by Network Time Foundation, 2026-02-19T20:27:26.121 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-02-19T20:27:26.121 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: corporation. Support and training for ntp-4 are 2026-02-19T20:27:26.121 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: available at https://www.nwtime.org/support 2026-02-19T20:27:26.121 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: ---------------------------------------------------- 2026-02-19T20:27:26.122 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: proto: precision = 0.030 usec (-25) 2026-02-19T20:27:26.122 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: basedate set to 2022-02-04 2026-02-19T20:27:26.122 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: gps base set to 2022-02-06 (week 2196) 2026-02-19T20:27:26.123 INFO:teuthology.orchestra.run.trial151.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-19T20:27:26.123 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-19T20:27:26.123 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: restrict ::: KOD does nothing without LIMITED. 2026-02-19T20:27:26.123 INFO:teuthology.orchestra.run.trial151.stderr:restrict ::: KOD does nothing without LIMITED. 2026-02-19T20:27:26.124 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: Listen and drop on 0 v6wildcard [::]:123 2026-02-19T20:27:26.124 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-02-19T20:27:26.124 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: Listen normally on 2 lo 127.0.0.1:123 2026-02-19T20:27:26.125 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: Listen normally on 3 enp1s0f0 10.20.193.151:123 2026-02-19T20:27:26.125 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: Listen normally on 4 lo [::1]:123 2026-02-19T20:27:26.125 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: Listen normally on 5 enp1s0f0 [fe80::925a:8ff:fe77:6454%2]:123 2026-02-19T20:27:26.125 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:26 ntpd[9417]: Listening on routing socket on fd #22 for interface updates 2026-02-19T20:27:27.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:26 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:27.108 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:26 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:27.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:27.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:27.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:27.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:27.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:27.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:27.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:27.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:27.189 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:27.190 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:27.190 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:27.190 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:27 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:29.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:29.122 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:29.122 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:29.122 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:29.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:29.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:29.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:29.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:29.189 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:29.189 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:29.189 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:29.189 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:29 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:30.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:29 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:30.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:29 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:30.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:30 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:30.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:30 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:30.125 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:30 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:30.125 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:30 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:30.125 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:30 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:30.125 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:30 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:30.149 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:29 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:30.150 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:29 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:30.150 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:29 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:30.150 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:29 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:31.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:30 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:31.108 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:30 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:31.108 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:30 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:31.108 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:30 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:31.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:31.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:31.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:31.122 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:31.122 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:31.122 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:31.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:30 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:31.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:30 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:31.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:30 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:31.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:30 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:31.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:31.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:31.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:31.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:31.183 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:31.183 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:31.184 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:31.184 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:31.189 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:31.189 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:31.189 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:31.189 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:32.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:32.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:32.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:32 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:32.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:32 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:32.125 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:32 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:32.125 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:32 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:32.125 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:32 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:32.125 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:32 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:32.148 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:32.149 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:32.149 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:31 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:32.149 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:31 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:33.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:32 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:33.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:32 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:33.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:33.121 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:33.122 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:33.122 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:33.122 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:33.122 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:33.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:32 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:33.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:32 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:33.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:32 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:33.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:32 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:33.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:33.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:33.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:33.141 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:33.141 INFO:teuthology.orchestra.run.trial151.stdout:19 Feb 20:27:33 ntpd[9417]: ntpd: time set +0.088189 s 2026-02-19T20:27:33.142 INFO:teuthology.orchestra.run.trial151.stdout:ntpd: time set +0.088189s 2026-02-19T20:27:33.142 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2026-02-19T20:27:33.142 INFO:teuthology.orchestra.run.trial151.stderr:19 Feb 20:27:33 ntpd[9417]: can't open /var/log/ntpstats/loopstats.20260219: Permission denied 2026-02-19T20:27:33.166 INFO:teuthology.orchestra.run.trial151.stdout:No association ID's returned 2026-02-19T20:27:34.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:33 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:34.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:33 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:34.148 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:33 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:34.148 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:33 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:34.148 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:33 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:34.148 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:33 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:35.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:34 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:35.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:34 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:35.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:34 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:35.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:34 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:35.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:34 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:35.126 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:34 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:36.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:35 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:36.107 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:35 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:36.148 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:35 ntpd[9417]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-19T20:27:36.149 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:35 ntpd[9417]: can't open /var/log/ntpstats/rawstats.20260219: Permission denied 2026-02-19T20:27:36.149 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:35 ntpd[9417]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-19T20:27:36.149 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:35 ntpd[9417]: can't open /var/log/ntpstats/peerstats.20260219: Permission denied 2026-02-19T20:27:36.149 INFO:teuthology.orchestra.run.trial013.stdout:19 Feb 20:27:36 ntpd[9417]: ntpd: time set +0.411298 s 2026-02-19T20:27:36.149 INFO:teuthology.orchestra.run.trial013.stdout:ntpd: time set +0.411298s 2026-02-19T20:27:36.150 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:36 ntpd[9417]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2026-02-19T20:27:36.150 INFO:teuthology.orchestra.run.trial013.stderr:19 Feb 20:27:36 ntpd[9417]: can't open /var/log/ntpstats/loopstats.20260219: Permission denied 2026-02-19T20:27:36.216 INFO:teuthology.orchestra.run.trial013.stdout: remote refid st t when poll reach delay offset jitter 2026-02-19T20:27:36.216 INFO:teuthology.orchestra.run.trial013.stdout:============================================================================== 2026-02-19T20:27:36.216 INFO:teuthology.orchestra.run.trial013.stdout: ntp.maxhost.io .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-19T20:27:36.216 INFO:teuthology.orchestra.run.trial013.stdout: ntp.maxhost.io .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-19T20:27:36.216 INFO:teuthology.orchestra.run.trial013.stdout: 23.155.72.147 ( .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-19T20:27:36.216 INFO:teuthology.orchestra.run.trial013.stdout: 104.233.211.205 .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-19T20:27:36.217 INFO:teuthology.run_tasks:Running task install... 2026-02-19T20:27:36.228 DEBUG:teuthology.task.install:project ceph 2026-02-19T20:27:36.229 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}} 2026-02-19T20:27:36.229 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'} 2026-02-19T20:27:36.229 INFO:teuthology.task.install:Using flavor: default 2026-02-19T20:27:36.232 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-osd', 'ceph-osd-classic', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-osd', 'ceph-osd-classic', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-19T20:27:36.233 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-19T20:27:36.233 DEBUG:teuthology.orchestra.run.trial013:> sudo apt-key list | grep Ceph 2026-02-19T20:27:36.233 DEBUG:teuthology.orchestra.run.trial151:> sudo apt-key list | grep Ceph 2026-02-19T20:27:36.283 INFO:teuthology.orchestra.run.trial151.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-02-19T20:27:36.301 INFO:teuthology.orchestra.run.trial013.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-02-19T20:27:36.312 INFO:teuthology.orchestra.run.trial151.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-02-19T20:27:36.312 INFO:teuthology.orchestra.run.trial151.stdout:uid [ unknown] Ceph.com (release key) 2026-02-19T20:27:36.313 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-osd, ceph-osd-classic, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-02-19T20:27:36.314 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=802bf111cd55e57f5842153d7ba2b8b964aa818a 2026-02-19T20:27:36.326 INFO:teuthology.orchestra.run.trial013.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-02-19T20:27:36.327 INFO:teuthology.orchestra.run.trial013.stdout:uid [ unknown] Ceph.com (release key) 2026-02-19T20:27:36.328 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-osd, ceph-osd-classic, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-02-19T20:27:36.328 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=802bf111cd55e57f5842153d7ba2b8b964aa818a 2026-02-19T20:27:36.508 INFO:teuthology.task.install.deb:Pulling from https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default/ 2026-02-19T20:27:36.508 INFO:teuthology.task.install.deb:Package version is 20.3.0-5336-g802bf111-1jammy 2026-02-19T20:27:36.521 INFO:teuthology.task.install.deb:Pulling from https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default/ 2026-02-19T20:27:36.521 INFO:teuthology.task.install.deb:Package version is 20.3.0-5336-g802bf111-1jammy 2026-02-19T20:27:36.776 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:27:36.777 DEBUG:teuthology.orchestra.run.trial151:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-02-19T20:27:36.779 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:27:36.779 DEBUG:teuthology.orchestra.run.trial013:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-02-19T20:27:36.788 DEBUG:teuthology.orchestra.run.trial151:> sudo apt-get update 2026-02-19T20:27:36.790 DEBUG:teuthology.orchestra.run.trial013:> sudo apt-get update 2026-02-19T20:27:36.982 INFO:teuthology.orchestra.run.trial151.stdout:Ign:1 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy InRelease 2026-02-19T20:27:36.988 INFO:teuthology.orchestra.run.trial013.stdout:Ign:1 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy InRelease 2026-02-19T20:27:36.993 INFO:teuthology.orchestra.run.trial151.stdout:Get:2 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy Release [7005 B] 2026-02-19T20:27:36.999 INFO:teuthology.orchestra.run.trial013.stdout:Get:2 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy Release [7005 B] 2026-02-19T20:27:37.005 INFO:teuthology.orchestra.run.trial151.stdout:Ign:3 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy Release.gpg 2026-02-19T20:27:37.059 INFO:teuthology.orchestra.run.trial151.stdout:Get:4 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 Packages [19.2 kB] 2026-02-19T20:27:37.059 INFO:teuthology.orchestra.run.trial013.stdout:Ign:3 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy Release.gpg 2026-02-19T20:27:37.106 INFO:teuthology.orchestra.run.trial013.stdout:Get:4 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 Packages [19.2 kB] 2026-02-19T20:27:37.226 INFO:teuthology.orchestra.run.trial013.stdout:Hit:5 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-19T20:27:37.293 INFO:teuthology.orchestra.run.trial013.stdout:Hit:6 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-19T20:27:37.366 INFO:teuthology.orchestra.run.trial013.stdout:Hit:7 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-19T20:27:37.439 INFO:teuthology.orchestra.run.trial013.stdout:Hit:8 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-19T20:27:37.535 INFO:teuthology.orchestra.run.trial151.stdout:Hit:5 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-19T20:27:37.585 INFO:teuthology.orchestra.run.trial013.stdout:Fetched 26.2 kB in 1s (41.6 kB/s) 2026-02-19T20:27:37.617 INFO:teuthology.orchestra.run.trial151.stdout:Hit:6 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-19T20:27:37.705 INFO:teuthology.orchestra.run.trial151.stdout:Hit:7 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-19T20:27:37.793 INFO:teuthology.orchestra.run.trial151.stdout:Hit:8 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-19T20:27:37.934 INFO:teuthology.orchestra.run.trial151.stdout:Fetched 26.2 kB in 1s (26.5 kB/s) 2026-02-19T20:27:38.122 INFO:teuthology.orchestra.run.trial013.stdout:Reading package lists... 2026-02-19T20:27:38.135 DEBUG:teuthology.orchestra.run.trial013:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=20.3.0-5336-g802bf111-1jammy cephadm=20.3.0-5336-g802bf111-1jammy ceph-mds=20.3.0-5336-g802bf111-1jammy ceph-mgr=20.3.0-5336-g802bf111-1jammy ceph-osd=20.3.0-5336-g802bf111-1jammy ceph-osd-classic=20.3.0-5336-g802bf111-1jammy ceph-common=20.3.0-5336-g802bf111-1jammy ceph-fuse=20.3.0-5336-g802bf111-1jammy ceph-test=20.3.0-5336-g802bf111-1jammy ceph-volume=20.3.0-5336-g802bf111-1jammy radosgw=20.3.0-5336-g802bf111-1jammy python3-rados=20.3.0-5336-g802bf111-1jammy python3-rgw=20.3.0-5336-g802bf111-1jammy python3-cephfs=20.3.0-5336-g802bf111-1jammy python3-rbd=20.3.0-5336-g802bf111-1jammy libcephfs2=20.3.0-5336-g802bf111-1jammy libcephfs-dev=20.3.0-5336-g802bf111-1jammy librados2=20.3.0-5336-g802bf111-1jammy librbd1=20.3.0-5336-g802bf111-1jammy rbd-fuse=20.3.0-5336-g802bf111-1jammy rbd-mirror=20.3.0-5336-g802bf111-1jammy 2026-02-19T20:27:38.168 INFO:teuthology.orchestra.run.trial013.stdout:Reading package lists... 2026-02-19T20:27:38.282 INFO:teuthology.orchestra.run.trial013.stdout:Building dependency tree... 2026-02-19T20:27:38.282 INFO:teuthology.orchestra.run.trial013.stdout:Reading state information... 2026-02-19T20:27:38.373 INFO:teuthology.orchestra.run.trial013.stdout:The following packages were automatically installed and are no longer required: 2026-02-19T20:27:38.373 INFO:teuthology.orchestra.run.trial013.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: sg3-utils-udev 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout:The following additional packages will be installed: 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon jq libcephfs-daemon 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: libcephfs-proxy2 libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 libnbd0 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: nvme-cli python-asyncssh-doc python3-asyncssh python3-cachetools 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: python3-google-auth python3-iniconfig python3-jaraco.classes 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: python3-joblib python3-kubernetes python3-natsort python3-pluggy 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: python3-portend python3-prettytable python3-psutil python3-py 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: python3-pygments python3-pytest python3-repoze.lru python3-requests-oauthlib 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: python3-routes python3-rsa python3-simplejson python3-sklearn 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: python3-sklearn-lib python3-tempora python3-threadpoolctl python3-toml 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: qttranslations5-l10n socat xmlstarlet 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout:Suggested packages: 2026-02-19T20:27:38.374 INFO:teuthology.orchestra.run.trial013.stdout: python3-influxdb liblua5.3-dev luarocks python-natsort-doc python-psutil-doc 2026-02-19T20:27:38.375 INFO:teuthology.orchestra.run.trial013.stdout: subversion python-pygments-doc ttf-bitstream-vera python3-paste python3-dap 2026-02-19T20:27:38.375 INFO:teuthology.orchestra.run.trial013.stdout: python-sklearn-doc ipython3 python-webob-doc 2026-02-19T20:27:38.375 INFO:teuthology.orchestra.run.trial013.stdout:Recommended packages: 2026-02-19T20:27:38.375 INFO:teuthology.orchestra.run.trial013.stdout: btrfs-tools 2026-02-19T20:27:38.409 INFO:teuthology.orchestra.run.trial013.stdout:The following NEW packages will be installed: 2026-02-19T20:27:38.409 INFO:teuthology.orchestra.run.trial013.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-02-19T20:27:38.409 INFO:teuthology.orchestra.run.trial013.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-02-19T20:27:38.409 INFO:teuthology.orchestra.run.trial013.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-osd-classic ceph-test 2026-02-19T20:27:38.409 INFO:teuthology.orchestra.run.trial013.stdout: ceph-volume cephadm jq libcephfs-daemon libcephfs-dev libcephfs-proxy2 2026-02-19T20:27:38.409 INFO:teuthology.orchestra.run.trial013.stdout: libcephfs2 libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 libnbd0 2026-02-19T20:27:38.409 INFO:teuthology.orchestra.run.trial013.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2026-02-19T20:27:38.409 INFO:teuthology.orchestra.run.trial013.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python3-asyncssh 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-natsort 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: python3-pygments python3-pytest python3-rados python3-rbd python3-repoze.lru 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: python3-simplejson python3-sklearn python3-sklearn-lib python3-tempora 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: python3-threadpoolctl python3-toml python3-wcwidth python3-webob 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2026-02-19T20:27:38.410 INFO:teuthology.orchestra.run.trial013.stdout: rbd-mirror socat xmlstarlet 2026-02-19T20:27:38.459 INFO:teuthology.orchestra.run.trial151.stdout:Reading package lists... 2026-02-19T20:27:38.472 DEBUG:teuthology.orchestra.run.trial151:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=20.3.0-5336-g802bf111-1jammy cephadm=20.3.0-5336-g802bf111-1jammy ceph-mds=20.3.0-5336-g802bf111-1jammy ceph-mgr=20.3.0-5336-g802bf111-1jammy ceph-osd=20.3.0-5336-g802bf111-1jammy ceph-osd-classic=20.3.0-5336-g802bf111-1jammy ceph-common=20.3.0-5336-g802bf111-1jammy ceph-fuse=20.3.0-5336-g802bf111-1jammy ceph-test=20.3.0-5336-g802bf111-1jammy ceph-volume=20.3.0-5336-g802bf111-1jammy radosgw=20.3.0-5336-g802bf111-1jammy python3-rados=20.3.0-5336-g802bf111-1jammy python3-rgw=20.3.0-5336-g802bf111-1jammy python3-cephfs=20.3.0-5336-g802bf111-1jammy python3-rbd=20.3.0-5336-g802bf111-1jammy libcephfs2=20.3.0-5336-g802bf111-1jammy libcephfs-dev=20.3.0-5336-g802bf111-1jammy librados2=20.3.0-5336-g802bf111-1jammy librbd1=20.3.0-5336-g802bf111-1jammy rbd-fuse=20.3.0-5336-g802bf111-1jammy rbd-mirror=20.3.0-5336-g802bf111-1jammy 2026-02-19T20:27:38.489 INFO:teuthology.orchestra.run.trial013.stdout:0 upgraded, 88 newly installed, 0 to remove and 13 not upgraded. 2026-02-19T20:27:38.489 INFO:teuthology.orchestra.run.trial013.stdout:Need to get 711 MB of archives. 2026-02-19T20:27:38.489 INFO:teuthology.orchestra.run.trial013.stdout:After this operation, 2700 MB of additional disk space will be used. 2026-02-19T20:27:38.489 INFO:teuthology.orchestra.run.trial013.stdout:Get:1 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 20.3.0-5336-g802bf111-1jammy [3927 kB] 2026-02-19T20:27:38.506 INFO:teuthology.orchestra.run.trial151.stdout:Reading package lists... 2026-02-19T20:27:38.618 INFO:teuthology.orchestra.run.trial013.stdout:Get:2 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 20.3.0-5336-g802bf111-1jammy [2913 kB] 2026-02-19T20:27:38.619 INFO:teuthology.orchestra.run.trial151.stdout:Building dependency tree... 2026-02-19T20:27:38.619 INFO:teuthology.orchestra.run.trial151.stdout:Reading state information... 2026-02-19T20:27:38.672 INFO:teuthology.orchestra.run.trial013.stdout:Get:3 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 20.3.0-5336-g802bf111-1jammy [883 kB] 2026-02-19T20:27:38.689 INFO:teuthology.orchestra.run.trial013.stdout:Get:4 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 20.3.0-5336-g802bf111-1jammy [359 kB] 2026-02-19T20:27:38.699 INFO:teuthology.orchestra.run.trial013.stdout:Get:5 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 20.3.0-5336-g802bf111-1jammy [32.8 kB] 2026-02-19T20:27:38.700 INFO:teuthology.orchestra.run.trial013.stdout:Get:6 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 20.3.0-5336-g802bf111-1jammy [225 kB] 2026-02-19T20:27:38.705 INFO:teuthology.orchestra.run.trial013.stdout:Get:7 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 20.3.0-5336-g802bf111-1jammy [88.4 kB] 2026-02-19T20:27:38.706 INFO:teuthology.orchestra.run.trial013.stdout:Get:8 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 20.3.0-5336-g802bf111-1jammy [339 kB] 2026-02-19T20:27:38.710 INFO:teuthology.orchestra.run.trial151.stdout:The following packages were automatically installed and are no longer required: 2026-02-19T20:27:38.710 INFO:teuthology.orchestra.run.trial151.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-19T20:27:38.710 INFO:teuthology.orchestra.run.trial151.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-19T20:27:38.710 INFO:teuthology.orchestra.run.trial151.stdout: sg3-utils-udev 2026-02-19T20:27:38.710 INFO:teuthology.orchestra.run.trial151.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout:The following additional packages will be installed: 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon jq libcephfs-daemon 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: libcephfs-proxy2 libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 libnbd0 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: nvme-cli python-asyncssh-doc python3-asyncssh python3-cachetools 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: python3-ceph-argparse python3-ceph-common python3-cheroot python3-cherrypy3 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: python3-google-auth python3-iniconfig python3-jaraco.classes 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: python3-joblib python3-kubernetes python3-natsort python3-pluggy 2026-02-19T20:27:38.711 INFO:teuthology.orchestra.run.trial151.stdout: python3-portend python3-prettytable python3-psutil python3-py 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout: python3-pygments python3-pytest python3-repoze.lru python3-requests-oauthlib 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout: python3-routes python3-rsa python3-simplejson python3-sklearn 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout: python3-sklearn-lib python3-tempora python3-threadpoolctl python3-toml 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout: python3-wcwidth python3-webob python3-websocket python3-zc.lockfile 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout: qttranslations5-l10n socat xmlstarlet 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout:Suggested packages: 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout: python3-influxdb liblua5.3-dev luarocks python-natsort-doc python-psutil-doc 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout: subversion python-pygments-doc ttf-bitstream-vera python3-paste python3-dap 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout: python-sklearn-doc ipython3 python-webob-doc 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout:Recommended packages: 2026-02-19T20:27:38.712 INFO:teuthology.orchestra.run.trial151.stdout: btrfs-tools 2026-02-19T20:27:38.713 INFO:teuthology.orchestra.run.trial013.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 20.3.0-5336-g802bf111-1jammy [9087 kB] 2026-02-19T20:27:38.744 INFO:teuthology.orchestra.run.trial151.stdout:The following NEW packages will be installed: 2026-02-19T20:27:38.744 INFO:teuthology.orchestra.run.trial151.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-osd-classic ceph-test 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: ceph-volume cephadm jq libcephfs-daemon libcephfs-dev libcephfs-proxy2 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: libcephfs2 libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 libnbd0 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python3-asyncssh 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-natsort 2026-02-19T20:27:38.745 INFO:teuthology.orchestra.run.trial151.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2026-02-19T20:27:38.746 INFO:teuthology.orchestra.run.trial151.stdout: python3-pygments python3-pytest python3-rados python3-rbd python3-repoze.lru 2026-02-19T20:27:38.746 INFO:teuthology.orchestra.run.trial151.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2026-02-19T20:27:38.746 INFO:teuthology.orchestra.run.trial151.stdout: python3-simplejson python3-sklearn python3-sklearn-lib python3-tempora 2026-02-19T20:27:38.746 INFO:teuthology.orchestra.run.trial151.stdout: python3-threadpoolctl python3-toml python3-wcwidth python3-webob 2026-02-19T20:27:38.746 INFO:teuthology.orchestra.run.trial151.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2026-02-19T20:27:38.746 INFO:teuthology.orchestra.run.trial151.stdout: rbd-mirror socat xmlstarlet 2026-02-19T20:27:38.843 INFO:teuthology.orchestra.run.trial013.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-02-19T20:27:38.883 INFO:teuthology.orchestra.run.trial151.stdout:0 upgraded, 88 newly installed, 0 to remove and 13 not upgraded. 2026-02-19T20:27:38.883 INFO:teuthology.orchestra.run.trial151.stdout:Need to get 711 MB of archives. 2026-02-19T20:27:38.883 INFO:teuthology.orchestra.run.trial151.stdout:After this operation, 2700 MB of additional disk space will be used. 2026-02-19T20:27:38.883 INFO:teuthology.orchestra.run.trial151.stdout:Get:1 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 20.3.0-5336-g802bf111-1jammy [3927 kB] 2026-02-19T20:27:38.906 INFO:teuthology.orchestra.run.trial013.stdout:Get:11 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 20.3.0-5336-g802bf111-1jammy [113 kB] 2026-02-19T20:27:38.906 INFO:teuthology.orchestra.run.trial013.stdout:Get:12 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 20.3.0-5336-g802bf111-1jammy [264 kB] 2026-02-19T20:27:38.913 INFO:teuthology.orchestra.run.trial013.stdout:Get:13 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 20.3.0-5336-g802bf111-1jammy [31.6 MB] 2026-02-19T20:27:39.163 INFO:teuthology.orchestra.run.trial151.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-02-19T20:27:39.274 INFO:teuthology.orchestra.run.trial013.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-02-19T20:27:39.287 INFO:teuthology.orchestra.run.trial013.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-02-19T20:27:39.377 INFO:teuthology.orchestra.run.trial013.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-02-19T20:27:39.562 INFO:teuthology.orchestra.run.trial013.stdout:Get:17 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 20.3.0-5336-g802bf111-1jammy [5000 kB] 2026-02-19T20:27:39.578 INFO:teuthology.orchestra.run.trial151.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-02-19T20:27:39.591 INFO:teuthology.orchestra.run.trial151.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-02-19T20:27:39.632 INFO:teuthology.orchestra.run.trial013.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-02-19T20:27:39.645 INFO:teuthology.orchestra.run.trial013.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-02-19T20:27:39.656 INFO:teuthology.orchestra.run.trial013.stdout:Get:20 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 20.3.0-5336-g802bf111-1jammy [293 kB] 2026-02-19T20:27:39.661 INFO:teuthology.orchestra.run.trial013.stdout:Get:21 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 20.3.0-5336-g802bf111-1jammy [127 kB] 2026-02-19T20:27:39.663 INFO:teuthology.orchestra.run.trial013.stdout:Get:22 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 20.3.0-5336-g802bf111-1jammy [923 kB] 2026-02-19T20:27:39.678 INFO:teuthology.orchestra.run.trial151.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-02-19T20:27:39.679 INFO:teuthology.orchestra.run.trial013.stdout:Get:23 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-02-19T20:27:39.681 INFO:teuthology.orchestra.run.trial013.stdout:Get:24 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 20.3.0-5336-g802bf111-1jammy [7543 kB] 2026-02-19T20:27:39.689 INFO:teuthology.orchestra.run.trial013.stdout:Get:25 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-02-19T20:27:39.691 INFO:teuthology.orchestra.run.trial013.stdout:Get:26 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-02-19T20:27:39.692 INFO:teuthology.orchestra.run.trial013.stdout:Get:27 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-02-19T20:27:39.694 INFO:teuthology.orchestra.run.trial013.stdout:Get:28 https://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-02-19T20:27:39.715 INFO:teuthology.orchestra.run.trial013.stdout:Get:29 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-02-19T20:27:39.716 INFO:teuthology.orchestra.run.trial013.stdout:Get:30 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-02-19T20:27:39.723 INFO:teuthology.orchestra.run.trial013.stdout:Get:31 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-02-19T20:27:39.808 INFO:teuthology.orchestra.run.trial013.stdout:Get:32 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-02-19T20:27:39.809 INFO:teuthology.orchestra.run.trial013.stdout:Get:33 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-02-19T20:27:39.809 INFO:teuthology.orchestra.run.trial013.stdout:Get:34 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-02-19T20:27:39.809 INFO:teuthology.orchestra.run.trial013.stdout:Get:35 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-02-19T20:27:39.810 INFO:teuthology.orchestra.run.trial013.stdout:Get:36 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-02-19T20:27:39.810 INFO:teuthology.orchestra.run.trial013.stdout:Get:37 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-02-19T20:27:39.810 INFO:teuthology.orchestra.run.trial013.stdout:Get:38 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-02-19T20:27:39.815 INFO:teuthology.orchestra.run.trial013.stdout:Get:39 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-02-19T20:27:39.815 INFO:teuthology.orchestra.run.trial013.stdout:Get:40 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-02-19T20:27:39.834 INFO:teuthology.orchestra.run.trial013.stdout:Get:41 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd-classic amd64 20.3.0-5336-g802bf111-1jammy [474 MB] 2026-02-19T20:27:39.904 INFO:teuthology.orchestra.run.trial013.stdout:Get:42 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-02-19T20:27:39.907 INFO:teuthology.orchestra.run.trial013.stdout:Get:43 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-02-19T20:27:39.907 INFO:teuthology.orchestra.run.trial013.stdout:Get:44 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-02-19T20:27:39.908 INFO:teuthology.orchestra.run.trial013.stdout:Get:45 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-02-19T20:27:39.923 INFO:teuthology.orchestra.run.trial151.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-02-19T20:27:39.936 INFO:teuthology.orchestra.run.trial151.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-02-19T20:27:39.939 INFO:teuthology.orchestra.run.trial013.stdout:Get:46 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-02-19T20:27:39.942 INFO:teuthology.orchestra.run.trial013.stdout:Get:47 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-02-19T20:27:39.942 INFO:teuthology.orchestra.run.trial013.stdout:Get:48 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-02-19T20:27:39.968 INFO:teuthology.orchestra.run.trial151.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-02-19T20:27:39.978 INFO:teuthology.orchestra.run.trial151.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-02-19T20:27:39.981 INFO:teuthology.orchestra.run.trial151.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-02-19T20:27:39.982 INFO:teuthology.orchestra.run.trial151.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-02-19T20:27:39.983 INFO:teuthology.orchestra.run.trial151.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-02-19T20:27:39.995 INFO:teuthology.orchestra.run.trial151.stdout:Get:13 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 20.3.0-5336-g802bf111-1jammy [2913 kB] 2026-02-19T20:27:40.003 INFO:teuthology.orchestra.run.trial151.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-02-19T20:27:40.004 INFO:teuthology.orchestra.run.trial151.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-02-19T20:27:40.011 INFO:teuthology.orchestra.run.trial151.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-02-19T20:27:40.015 INFO:teuthology.orchestra.run.trial013.stdout:Get:49 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-02-19T20:27:40.015 INFO:teuthology.orchestra.run.trial013.stdout:Get:50 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-02-19T20:27:40.016 INFO:teuthology.orchestra.run.trial013.stdout:Get:51 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-02-19T20:27:40.016 INFO:teuthology.orchestra.run.trial013.stdout:Get:52 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-02-19T20:27:40.017 INFO:teuthology.orchestra.run.trial013.stdout:Get:53 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-02-19T20:27:40.017 INFO:teuthology.orchestra.run.trial013.stdout:Get:54 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-02-19T20:27:40.093 INFO:teuthology.orchestra.run.trial013.stdout:Get:55 https://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-02-19T20:27:40.094 INFO:teuthology.orchestra.run.trial151.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-02-19T20:27:40.094 INFO:teuthology.orchestra.run.trial151.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-02-19T20:27:40.094 INFO:teuthology.orchestra.run.trial151.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-02-19T20:27:40.094 INFO:teuthology.orchestra.run.trial151.stdout:Get:20 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-02-19T20:27:40.094 INFO:teuthology.orchestra.run.trial151.stdout:Get:21 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-02-19T20:27:40.095 INFO:teuthology.orchestra.run.trial013.stdout:Get:56 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-02-19T20:27:40.095 INFO:teuthology.orchestra.run.trial151.stdout:Get:22 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-02-19T20:27:40.095 INFO:teuthology.orchestra.run.trial151.stdout:Get:23 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-02-19T20:27:40.096 INFO:teuthology.orchestra.run.trial013.stdout:Get:57 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-02-19T20:27:40.099 INFO:teuthology.orchestra.run.trial151.stdout:Get:24 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-02-19T20:27:40.100 INFO:teuthology.orchestra.run.trial151.stdout:Get:25 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-02-19T20:27:40.184 INFO:teuthology.orchestra.run.trial013.stdout:Get:58 https://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-02-19T20:27:40.185 INFO:teuthology.orchestra.run.trial151.stdout:Get:26 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-02-19T20:27:40.188 INFO:teuthology.orchestra.run.trial151.stdout:Get:27 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-02-19T20:27:40.188 INFO:teuthology.orchestra.run.trial013.stdout:Get:59 https://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-02-19T20:27:40.188 INFO:teuthology.orchestra.run.trial151.stdout:Get:28 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-02-19T20:27:40.189 INFO:teuthology.orchestra.run.trial151.stdout:Get:29 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-02-19T20:27:40.190 INFO:teuthology.orchestra.run.trial013.stdout:Get:60 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-02-19T20:27:40.196 INFO:teuthology.orchestra.run.trial013.stdout:Get:61 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-02-19T20:27:40.198 INFO:teuthology.orchestra.run.trial013.stdout:Get:62 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-02-19T20:27:40.199 INFO:teuthology.orchestra.run.trial013.stdout:Get:63 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-02-19T20:27:40.199 INFO:teuthology.orchestra.run.trial013.stdout:Get:64 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-02-19T20:27:40.219 INFO:teuthology.orchestra.run.trial151.stdout:Get:30 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-02-19T20:27:40.222 INFO:teuthology.orchestra.run.trial151.stdout:Get:31 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-02-19T20:27:40.222 INFO:teuthology.orchestra.run.trial151.stdout:Get:32 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-02-19T20:27:40.276 INFO:teuthology.orchestra.run.trial013.stdout:Get:65 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-02-19T20:27:40.277 INFO:teuthology.orchestra.run.trial013.stdout:Get:66 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-02-19T20:27:40.285 INFO:teuthology.orchestra.run.trial013.stdout:Get:67 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-02-19T20:27:40.293 INFO:teuthology.orchestra.run.trial151.stdout:Get:33 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-02-19T20:27:40.294 INFO:teuthology.orchestra.run.trial151.stdout:Get:34 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-02-19T20:27:40.294 INFO:teuthology.orchestra.run.trial151.stdout:Get:35 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-02-19T20:27:40.294 INFO:teuthology.orchestra.run.trial151.stdout:Get:36 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-02-19T20:27:40.294 INFO:teuthology.orchestra.run.trial151.stdout:Get:37 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-02-19T20:27:40.295 INFO:teuthology.orchestra.run.trial151.stdout:Get:38 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-02-19T20:27:40.367 INFO:teuthology.orchestra.run.trial151.stdout:Get:39 https://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-02-19T20:27:40.368 INFO:teuthology.orchestra.run.trial013.stdout:Get:68 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-02-19T20:27:40.369 INFO:teuthology.orchestra.run.trial151.stdout:Get:40 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-02-19T20:27:40.370 INFO:teuthology.orchestra.run.trial013.stdout:Get:69 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-02-19T20:27:40.370 INFO:teuthology.orchestra.run.trial151.stdout:Get:41 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-02-19T20:27:40.371 INFO:teuthology.orchestra.run.trial013.stdout:Get:70 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-02-19T20:27:40.372 INFO:teuthology.orchestra.run.trial013.stdout:Get:71 https://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-02-19T20:27:40.456 INFO:teuthology.orchestra.run.trial151.stdout:Get:42 https://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-02-19T20:27:40.459 INFO:teuthology.orchestra.run.trial151.stdout:Get:43 https://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-02-19T20:27:40.461 INFO:teuthology.orchestra.run.trial151.stdout:Get:44 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-02-19T20:27:40.466 INFO:teuthology.orchestra.run.trial151.stdout:Get:45 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-02-19T20:27:40.469 INFO:teuthology.orchestra.run.trial151.stdout:Get:46 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-02-19T20:27:40.470 INFO:teuthology.orchestra.run.trial151.stdout:Get:47 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-02-19T20:27:40.470 INFO:teuthology.orchestra.run.trial151.stdout:Get:48 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-02-19T20:27:40.499 INFO:teuthology.orchestra.run.trial151.stdout:Get:49 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 20.3.0-5336-g802bf111-1jammy [883 kB] 2026-02-19T20:27:40.544 INFO:teuthology.orchestra.run.trial151.stdout:Get:50 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-02-19T20:27:40.544 INFO:teuthology.orchestra.run.trial151.stdout:Get:51 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-02-19T20:27:40.552 INFO:teuthology.orchestra.run.trial151.stdout:Get:52 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-02-19T20:27:40.628 INFO:teuthology.orchestra.run.trial151.stdout:Get:53 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 20.3.0-5336-g802bf111-1jammy [359 kB] 2026-02-19T20:27:40.632 INFO:teuthology.orchestra.run.trial151.stdout:Get:54 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-02-19T20:27:40.634 INFO:teuthology.orchestra.run.trial151.stdout:Get:55 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-02-19T20:27:40.634 INFO:teuthology.orchestra.run.trial151.stdout:Get:56 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-02-19T20:27:40.635 INFO:teuthology.orchestra.run.trial151.stdout:Get:57 https://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-02-19T20:27:40.691 INFO:teuthology.orchestra.run.trial151.stdout:Get:58 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 20.3.0-5336-g802bf111-1jammy [32.8 kB] 2026-02-19T20:27:40.692 INFO:teuthology.orchestra.run.trial151.stdout:Get:59 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 20.3.0-5336-g802bf111-1jammy [225 kB] 2026-02-19T20:27:40.696 INFO:teuthology.orchestra.run.trial151.stdout:Get:60 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 20.3.0-5336-g802bf111-1jammy [88.4 kB] 2026-02-19T20:27:40.697 INFO:teuthology.orchestra.run.trial151.stdout:Get:61 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 20.3.0-5336-g802bf111-1jammy [339 kB] 2026-02-19T20:27:40.760 INFO:teuthology.orchestra.run.trial151.stdout:Get:62 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 20.3.0-5336-g802bf111-1jammy [9087 kB] 2026-02-19T20:27:41.609 INFO:teuthology.orchestra.run.trial151.stdout:Get:63 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 20.3.0-5336-g802bf111-1jammy [113 kB] 2026-02-19T20:27:41.610 INFO:teuthology.orchestra.run.trial151.stdout:Get:64 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 20.3.0-5336-g802bf111-1jammy [264 kB] 2026-02-19T20:27:41.615 INFO:teuthology.orchestra.run.trial151.stdout:Get:65 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 20.3.0-5336-g802bf111-1jammy [31.6 MB] 2026-02-19T20:27:43.958 INFO:teuthology.orchestra.run.trial151.stdout:Get:66 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 20.3.0-5336-g802bf111-1jammy [5000 kB] 2026-02-19T20:27:44.326 INFO:teuthology.orchestra.run.trial151.stdout:Get:67 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 20.3.0-5336-g802bf111-1jammy [293 kB] 2026-02-19T20:27:44.372 INFO:teuthology.orchestra.run.trial151.stdout:Get:68 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 20.3.0-5336-g802bf111-1jammy [127 kB] 2026-02-19T20:27:44.373 INFO:teuthology.orchestra.run.trial151.stdout:Get:69 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 20.3.0-5336-g802bf111-1jammy [923 kB] 2026-02-19T20:27:44.432 INFO:teuthology.orchestra.run.trial151.stdout:Get:70 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 20.3.0-5336-g802bf111-1jammy [7543 kB] 2026-02-19T20:27:45.013 INFO:teuthology.orchestra.run.trial151.stdout:Get:71 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd-classic amd64 20.3.0-5336-g802bf111-1jammy [474 MB] 2026-02-19T20:27:51.648 INFO:teuthology.orchestra.run.trial013.stdout:Get:72 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 20.3.0-5336-g802bf111-1jammy [181 kB] 2026-02-19T20:27:51.649 INFO:teuthology.orchestra.run.trial013.stdout:Get:73 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 20.3.0-5336-g802bf111-1jammy [14.0 kB] 2026-02-19T20:27:51.649 INFO:teuthology.orchestra.run.trial013.stdout:Get:74 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 20.3.0-5336-g802bf111-1jammy [1033 kB] 2026-02-19T20:27:51.654 INFO:teuthology.orchestra.run.trial013.stdout:Get:75 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 20.3.0-5336-g802bf111-1jammy [2441 kB] 2026-02-19T20:27:51.667 INFO:teuthology.orchestra.run.trial013.stdout:Get:76 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 20.3.0-5336-g802bf111-1jammy [1069 kB] 2026-02-19T20:27:51.673 INFO:teuthology.orchestra.run.trial013.stdout:Get:77 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 20.3.0-5336-g802bf111-1jammy [190 kB] 2026-02-19T20:27:51.674 INFO:teuthology.orchestra.run.trial013.stdout:Get:78 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 20.3.0-5336-g802bf111-1jammy [54.8 MB] 2026-02-19T20:27:53.207 INFO:teuthology.orchestra.run.trial013.stdout:Get:79 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 20.3.0-5336-g802bf111-1jammy [8626 kB] 2026-02-19T20:27:53.504 INFO:teuthology.orchestra.run.trial013.stdout:Get:80 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 20.3.0-5336-g802bf111-1jammy [14.2 kB] 2026-02-19T20:27:53.505 INFO:teuthology.orchestra.run.trial013.stdout:Get:81 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 20.3.0-5336-g802bf111-1jammy [68.7 MB] 2026-02-19T20:27:56.023 INFO:teuthology.orchestra.run.trial013.stdout:Get:82 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 20.3.0-5336-g802bf111-1jammy [137 kB] 2026-02-19T20:27:56.024 INFO:teuthology.orchestra.run.trial013.stdout:Get:83 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-daemon amd64 20.3.0-5336-g802bf111-1jammy [45.8 kB] 2026-02-19T20:27:56.026 INFO:teuthology.orchestra.run.trial013.stdout:Get:84 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-proxy2 amd64 20.3.0-5336-g802bf111-1jammy [32.4 kB] 2026-02-19T20:27:56.026 INFO:teuthology.orchestra.run.trial013.stdout:Get:85 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 20.3.0-5336-g802bf111-1jammy [44.2 kB] 2026-02-19T20:27:56.028 INFO:teuthology.orchestra.run.trial013.stdout:Get:86 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 20.3.0-5336-g802bf111-1jammy [18.3 MB] 2026-02-19T20:27:56.731 INFO:teuthology.orchestra.run.trial013.stdout:Get:87 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 20.3.0-5336-g802bf111-1jammy [94.3 kB] 2026-02-19T20:27:56.734 INFO:teuthology.orchestra.run.trial013.stdout:Get:88 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 20.3.0-5336-g802bf111-1jammy [2928 kB] 2026-02-19T20:27:57.126 INFO:teuthology.orchestra.run.trial013.stdout:Fetched 711 MB in 18s (38.6 MB/s) 2026-02-19T20:27:57.145 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-02-19T20:27:57.302 INFO:teuthology.orchestra.run.trial013.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 167442 files and directories currently installed.) 2026-02-19T20:27:57.304 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../00-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-02-19T20:27:57.305 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-19T20:27:57.326 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-02-19T20:27:57.338 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../01-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-02-19T20:27:57.339 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-19T20:27:57.356 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-02-19T20:27:57.368 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../02-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-02-19T20:27:57.369 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-19T20:27:57.388 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-02-19T20:27:57.396 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../03-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-19T20:27:57.401 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:27:57.433 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-02-19T20:27:57.441 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../04-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-19T20:27:57.441 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:27:57.458 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-02-19T20:27:57.465 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../05-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-19T20:27:57.466 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:27:57.486 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-02-19T20:27:57.494 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../06-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-02-19T20:27:57.494 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-19T20:27:57.511 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package librados2. 2026-02-19T20:27:57.519 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../07-librados2_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:57.519 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking librados2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:57.570 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libnbd0. 2026-02-19T20:27:57.584 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../08-libnbd0_1.10.5-1_amd64.deb ... 2026-02-19T20:27:57.585 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-02-19T20:27:57.594 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package librbd1. 2026-02-19T20:27:57.601 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../09-librbd1_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:57.601 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking librbd1 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:57.639 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libcephfs2. 2026-02-19T20:27:57.647 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../10-libcephfs2_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:57.648 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libcephfs2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:57.667 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-rados. 2026-02-19T20:27:57.675 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../11-python3-rados_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:57.676 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-rados (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:57.695 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-02-19T20:27:57.703 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../12-python3-ceph-argparse_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:27:57.703 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-ceph-argparse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:57.717 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-cephfs. 2026-02-19T20:27:57.724 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../13-python3-cephfs_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:57.725 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-cephfs (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:57.741 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-ceph-common. 2026-02-19T20:27:57.749 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../14-python3-ceph-common_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:27:57.749 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-ceph-common (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:57.767 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-wcwidth. 2026-02-19T20:27:57.775 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../15-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-02-19T20:27:57.776 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-19T20:27:57.791 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-prettytable. 2026-02-19T20:27:57.799 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../16-python3-prettytable_2.5.0-2_all.deb ... 2026-02-19T20:27:57.799 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-02-19T20:27:57.814 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-rbd. 2026-02-19T20:27:57.822 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../17-python3-rbd_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:57.823 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-rbd (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:57.840 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-02-19T20:27:57.848 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../18-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-02-19T20:27:57.848 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-19T20:27:57.866 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package librgw2. 2026-02-19T20:27:57.874 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../19-librgw2_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:57.874 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking librgw2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:57.973 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-rgw. 2026-02-19T20:27:57.982 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../20-python3-rgw_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:57.982 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-rgw (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:57.997 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package liboath0:amd64. 2026-02-19T20:27:58.005 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../21-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-02-19T20:27:58.006 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-19T20:27:58.019 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libradosstriper1. 2026-02-19T20:27:58.028 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../22-libradosstriper1_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:58.028 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libradosstriper1 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:58.044 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-common. 2026-02-19T20:27:58.052 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../23-ceph-common_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:58.053 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-common (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:58.309 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-base. 2026-02-19T20:27:58.318 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../24-ceph-base_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:58.323 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-base (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:58.381 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-02-19T20:27:58.389 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../25-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-02-19T20:27:58.390 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-02-19T20:27:58.404 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-cheroot. 2026-02-19T20:27:58.412 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../26-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-02-19T20:27:58.412 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-19T20:27:58.429 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-02-19T20:27:58.437 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../27-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-02-19T20:27:58.438 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-02-19T20:27:58.452 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-jaraco.text. 2026-02-19T20:27:58.461 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../28-python3-jaraco.text_3.6.0-2_all.deb ... 2026-02-19T20:27:58.461 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-02-19T20:27:58.475 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-02-19T20:27:58.483 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../29-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-02-19T20:27:58.483 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-02-19T20:27:58.498 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-tempora. 2026-02-19T20:27:58.507 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../30-python3-tempora_4.1.2-1_all.deb ... 2026-02-19T20:27:58.507 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-02-19T20:27:58.522 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-portend. 2026-02-19T20:27:58.530 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../31-python3-portend_3.0.0-1_all.deb ... 2026-02-19T20:27:58.530 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-02-19T20:27:58.545 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-02-19T20:27:58.554 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../32-python3-zc.lockfile_2.0-1_all.deb ... 2026-02-19T20:27:58.554 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-02-19T20:27:58.568 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-cherrypy3. 2026-02-19T20:27:58.576 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../33-python3-cherrypy3_18.6.1-4_all.deb ... 2026-02-19T20:27:58.576 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-02-19T20:27:58.599 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-natsort. 2026-02-19T20:27:58.607 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../34-python3-natsort_8.0.2-1_all.deb ... 2026-02-19T20:27:58.608 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-02-19T20:27:58.623 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-02-19T20:27:58.631 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../35-ceph-mgr-modules-core_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:27:58.631 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-mgr-modules-core (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:58.662 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-02-19T20:27:58.670 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../36-libsqlite3-mod-ceph_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:58.670 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libsqlite3-mod-ceph (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:58.685 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-mgr. 2026-02-19T20:27:58.693 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../37-ceph-mgr_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:58.693 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-mgr (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:58.713 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-mon. 2026-02-19T20:27:58.721 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../38-ceph-mon_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:58.722 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-mon (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:27:58.787 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libfuse2:amd64. 2026-02-19T20:27:58.795 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../39-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-02-19T20:27:58.795 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-19T20:27:58.811 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-osd-classic. 2026-02-19T20:27:58.819 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../40-ceph-osd-classic_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:27:58.820 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-osd-classic (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:02.848 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-osd. 2026-02-19T20:28:02.860 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../41-ceph-osd_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:02.861 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-osd (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:02.880 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph. 2026-02-19T20:28:02.892 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../42-ceph_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:02.893 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:02.909 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-fuse. 2026-02-19T20:28:02.921 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../43-ceph-fuse_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:02.922 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-fuse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:02.946 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-mds. 2026-02-19T20:28:02.956 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../44-ceph-mds_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:02.956 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-mds (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:02.995 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package cephadm. 2026-02-19T20:28:03.003 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../45-cephadm_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:03.004 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking cephadm (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:03.019 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-asyncssh. 2026-02-19T20:28:03.028 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../46-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-02-19T20:28:03.028 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-19T20:28:03.049 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-02-19T20:28:03.058 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../47-ceph-mgr-cephadm_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:03.058 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-mgr-cephadm (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:03.079 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-repoze.lru. 2026-02-19T20:28:03.088 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../48-python3-repoze.lru_0.7-2_all.deb ... 2026-02-19T20:28:03.088 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-02-19T20:28:03.103 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-routes. 2026-02-19T20:28:03.111 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../49-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-02-19T20:28:03.111 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-02-19T20:28:03.132 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-02-19T20:28:03.140 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../50-ceph-mgr-dashboard_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:03.140 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-mgr-dashboard (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:03.832 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-02-19T20:28:03.841 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../51-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-02-19T20:28:03.842 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-19T20:28:03.880 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-joblib. 2026-02-19T20:28:03.889 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../52-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-02-19T20:28:03.889 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-02-19T20:28:03.916 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-02-19T20:28:03.925 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../53-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-02-19T20:28:03.925 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-02-19T20:28:03.940 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-sklearn. 2026-02-19T20:28:03.948 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../54-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-02-19T20:28:03.948 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-19T20:28:04.032 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-02-19T20:28:04.041 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../55-ceph-mgr-diskprediction-local_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:04.041 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-mgr-diskprediction-local (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:04.197 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-cachetools. 2026-02-19T20:28:04.205 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../56-python3-cachetools_5.0.0-1_all.deb ... 2026-02-19T20:28:04.206 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-02-19T20:28:04.220 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-rsa. 2026-02-19T20:28:04.228 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../57-python3-rsa_4.8-1_all.deb ... 2026-02-19T20:28:04.229 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-rsa (4.8-1) ... 2026-02-19T20:28:04.245 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-google-auth. 2026-02-19T20:28:04.253 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../58-python3-google-auth_1.5.1-3_all.deb ... 2026-02-19T20:28:04.254 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-02-19T20:28:04.271 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-02-19T20:28:04.279 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../59-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-02-19T20:28:04.280 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-19T20:28:04.296 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-websocket. 2026-02-19T20:28:04.304 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../60-python3-websocket_1.2.3-1_all.deb ... 2026-02-19T20:28:04.305 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-02-19T20:28:04.322 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-kubernetes. 2026-02-19T20:28:04.330 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../61-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-02-19T20:28:04.330 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-19T20:28:04.423 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-02-19T20:28:04.432 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../62-ceph-mgr-k8sevents_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:04.432 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-mgr-k8sevents (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:04.446 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libonig5:amd64. 2026-02-19T20:28:04.454 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../63-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-02-19T20:28:04.454 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-19T20:28:04.470 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libjq1:amd64. 2026-02-19T20:28:04.478 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../64-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-19T20:28:04.479 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-19T20:28:04.493 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package jq. 2026-02-19T20:28:04.502 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../65-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-19T20:28:04.502 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-02-19T20:28:04.517 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package socat. 2026-02-19T20:28:04.525 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../66-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-02-19T20:28:04.526 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-02-19T20:28:04.545 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package xmlstarlet. 2026-02-19T20:28:04.553 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../67-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-02-19T20:28:04.553 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-02-19T20:28:04.590 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-test. 2026-02-19T20:28:04.598 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../68-ceph-test_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:04.598 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-test (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:05.242 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package ceph-volume. 2026-02-19T20:28:05.250 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../69-ceph-volume_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:05.250 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking ceph-volume (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:05.274 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libcephfs-daemon. 2026-02-19T20:28:05.283 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../70-libcephfs-daemon_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:05.283 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libcephfs-daemon (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:05.298 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libcephfs-proxy2. 2026-02-19T20:28:05.306 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../71-libcephfs-proxy2_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:05.306 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libcephfs-proxy2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:05.322 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package libcephfs-dev. 2026-02-19T20:28:05.330 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../72-libcephfs-dev_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:05.331 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking libcephfs-dev (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:05.347 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package nvme-cli. 2026-02-19T20:28:05.355 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../73-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-02-19T20:28:05.355 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-02-19T20:28:05.387 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-02-19T20:28:05.396 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../74-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-02-19T20:28:05.396 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-19T20:28:05.425 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-iniconfig. 2026-02-19T20:28:05.434 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../75-python3-iniconfig_1.1.1-2_all.deb ... 2026-02-19T20:28:05.434 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-02-19T20:28:05.449 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-pluggy. 2026-02-19T20:28:05.457 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../76-python3-pluggy_0.13.0-7.1_all.deb ... 2026-02-19T20:28:05.458 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-02-19T20:28:05.473 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-psutil. 2026-02-19T20:28:05.482 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../77-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-02-19T20:28:05.482 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-02-19T20:28:05.501 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-py. 2026-02-19T20:28:05.509 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../78-python3-py_1.10.0-1_all.deb ... 2026-02-19T20:28:05.510 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-py (1.10.0-1) ... 2026-02-19T20:28:05.530 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-pygments. 2026-02-19T20:28:05.531 INFO:teuthology.orchestra.run.trial151.stdout:Get:72 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 20.3.0-5336-g802bf111-1jammy [181 kB] 2026-02-19T20:28:05.532 INFO:teuthology.orchestra.run.trial151.stdout:Get:73 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 20.3.0-5336-g802bf111-1jammy [14.0 kB] 2026-02-19T20:28:05.532 INFO:teuthology.orchestra.run.trial151.stdout:Get:74 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 20.3.0-5336-g802bf111-1jammy [1033 kB] 2026-02-19T20:28:05.538 INFO:teuthology.orchestra.run.trial151.stdout:Get:75 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 20.3.0-5336-g802bf111-1jammy [2441 kB] 2026-02-19T20:28:05.538 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../79-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-02-19T20:28:05.538 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-19T20:28:05.550 INFO:teuthology.orchestra.run.trial151.stdout:Get:76 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 20.3.0-5336-g802bf111-1jammy [1069 kB] 2026-02-19T20:28:05.556 INFO:teuthology.orchestra.run.trial151.stdout:Get:77 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 20.3.0-5336-g802bf111-1jammy [190 kB] 2026-02-19T20:28:05.557 INFO:teuthology.orchestra.run.trial151.stdout:Get:78 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 20.3.0-5336-g802bf111-1jammy [54.8 MB] 2026-02-19T20:28:05.580 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-toml. 2026-02-19T20:28:05.588 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../80-python3-toml_0.10.2-1_all.deb ... 2026-02-19T20:28:05.589 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-02-19T20:28:05.604 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-pytest. 2026-02-19T20:28:05.612 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../81-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-02-19T20:28:05.612 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-02-19T20:28:05.634 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-simplejson. 2026-02-19T20:28:05.642 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../82-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-02-19T20:28:05.642 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-02-19T20:28:05.667 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package python3-webob. 2026-02-19T20:28:05.675 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../83-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-02-19T20:28:05.675 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-19T20:28:05.693 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-02-19T20:28:05.701 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../84-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-02-19T20:28:05.702 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-02-19T20:28:05.768 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package radosgw. 2026-02-19T20:28:05.777 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../85-radosgw_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:05.777 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking radosgw (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:05.941 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package rbd-fuse. 2026-02-19T20:28:05.950 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../86-rbd-fuse_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:05.950 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking rbd-fuse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:05.966 INFO:teuthology.orchestra.run.trial013.stdout:Selecting previously unselected package rbd-mirror. 2026-02-19T20:28:05.975 INFO:teuthology.orchestra.run.trial013.stdout:Preparing to unpack .../87-rbd-mirror_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:05.975 INFO:teuthology.orchestra.run.trial013.stdout:Unpacking rbd-mirror (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:06.029 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-02-19T20:28:06.094 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-19T20:28:06.095 INFO:teuthology.orchestra.run.trial013.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-02-19T20:28:06.166 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-02-19T20:28:06.446 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-02-19T20:28:06.449 INFO:teuthology.orchestra.run.trial151.stdout:Get:79 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 20.3.0-5336-g802bf111-1jammy [8626 kB] 2026-02-19T20:28:06.583 INFO:teuthology.orchestra.run.trial151.stdout:Get:80 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 20.3.0-5336-g802bf111-1jammy [14.2 kB] 2026-02-19T20:28:06.583 INFO:teuthology.orchestra.run.trial151.stdout:Get:81 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 20.3.0-5336-g802bf111-1jammy [68.7 MB] 2026-02-19T20:28:06.836 INFO:teuthology.orchestra.run.trial013.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-02-19T20:28:06.848 INFO:teuthology.orchestra.run.trial013.stdout:Setting up cephadm (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:06.890 INFO:teuthology.orchestra.run.trial013.stdout:Adding system user cephadm....done 2026-02-19T20:28:06.895 INFO:teuthology.orchestra.run.trial013.stdout:usermod: no changes 2026-02-19T20:28:06.901 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-02-19T20:28:06.966 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-19T20:28:06.967 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-02-19T20:28:07.039 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-02-19T20:28:07.114 INFO:teuthology.orchestra.run.trial013.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-19T20:28:07.115 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-py (1.10.0-1) ... 2026-02-19T20:28:07.210 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-02-19T20:28:07.332 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-02-19T20:28:07.406 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-02-19T20:28:07.481 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-ceph-argparse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:07.550 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-19T20:28:07.551 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-02-19T20:28:07.552 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-19T20:28:07.553 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-19T20:28:07.554 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-02-19T20:28:07.676 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-02-19T20:28:07.716 INFO:teuthology.orchestra.run.trial151.stdout:Get:82 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 20.3.0-5336-g802bf111-1jammy [137 kB] 2026-02-19T20:28:07.717 INFO:teuthology.orchestra.run.trial151.stdout:Get:83 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-daemon amd64 20.3.0-5336-g802bf111-1jammy [45.8 kB] 2026-02-19T20:28:07.717 INFO:teuthology.orchestra.run.trial151.stdout:Get:84 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-proxy2 amd64 20.3.0-5336-g802bf111-1jammy [32.4 kB] 2026-02-19T20:28:07.718 INFO:teuthology.orchestra.run.trial151.stdout:Get:85 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 20.3.0-5336-g802bf111-1jammy [44.2 kB] 2026-02-19T20:28:07.718 INFO:teuthology.orchestra.run.trial151.stdout:Get:86 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 20.3.0-5336-g802bf111-1jammy [18.3 MB] 2026-02-19T20:28:07.752 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libcephfs-proxy2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:07.753 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-02-19T20:28:07.829 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-02-19T20:28:07.920 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-19T20:28:08.012 INFO:teuthology.orchestra.run.trial151.stdout:Get:87 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 20.3.0-5336-g802bf111-1jammy [94.3 kB] 2026-02-19T20:28:08.013 INFO:teuthology.orchestra.run.trial151.stdout:Get:88 https://3.chacra.ceph.com/r/ceph/wip-yuri-testing-2026-02-18-1820/802bf111cd55e57f5842153d7ba2b8b964aa818a/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 20.3.0-5336-g802bf111-1jammy [2928 kB] 2026-02-19T20:28:08.165 INFO:teuthology.orchestra.run.trial013.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-02-19T20:28:08.166 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-19T20:28:08.261 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-19T20:28:08.338 INFO:teuthology.orchestra.run.trial151.stdout:Fetched 711 MB in 29s (24.3 MB/s) 2026-02-19T20:28:08.355 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-02-19T20:28:08.386 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-19T20:28:08.468 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-02-19T20:28:08.515 INFO:teuthology.orchestra.run.trial151.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 167442 files and directories currently installed.) 2026-02-19T20:28:08.517 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../00-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-02-19T20:28:08.519 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-19T20:28:08.539 INFO:teuthology.orchestra.run.trial013.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-02-19T20:28:08.540 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-ceph-common (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:08.541 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-02-19T20:28:08.553 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../01-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-02-19T20:28:08.554 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-19T20:28:08.571 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-02-19T20:28:08.583 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../02-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-02-19T20:28:08.584 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-19T20:28:08.603 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-02-19T20:28:08.611 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../03-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-19T20:28:08.616 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:28:08.629 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-19T20:28:08.647 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-02-19T20:28:08.655 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../04-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-19T20:28:08.656 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:28:08.671 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-02-19T20:28:08.680 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../05-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-19T20:28:08.680 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:28:08.699 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-02-19T20:28:08.707 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../06-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-02-19T20:28:08.707 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-19T20:28:08.722 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package librados2. 2026-02-19T20:28:08.730 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../07-librados2_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:08.731 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking librados2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:08.774 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libnbd0. 2026-02-19T20:28:08.783 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../08-libnbd0_1.10.5-1_amd64.deb ... 2026-02-19T20:28:08.783 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-02-19T20:28:08.798 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package librbd1. 2026-02-19T20:28:08.806 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../09-librbd1_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:08.806 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking librbd1 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:08.842 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libcephfs2. 2026-02-19T20:28:08.851 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../10-libcephfs2_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:08.851 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libcephfs2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:08.871 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-rados. 2026-02-19T20:28:08.879 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../11-python3-rados_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:08.879 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-rados (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:08.896 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-02-19T20:28:08.904 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../12-python3-ceph-argparse_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:08.905 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-ceph-argparse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:08.918 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-cephfs. 2026-02-19T20:28:08.926 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../13-python3-cephfs_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:08.995 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-cephfs (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:09.027 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-ceph-common. 2026-02-19T20:28:09.040 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../14-python3-ceph-common_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:09.040 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-ceph-common (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:09.063 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-wcwidth. 2026-02-19T20:28:09.076 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../15-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-02-19T20:28:09.076 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-19T20:28:09.093 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-prettytable. 2026-02-19T20:28:09.106 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../16-python3-prettytable_2.5.0-2_all.deb ... 2026-02-19T20:28:09.106 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-02-19T20:28:09.121 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:28:09.121 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-rbd. 2026-02-19T20:28:09.126 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-toml (0.10.2-1) ... 2026-02-19T20:28:09.133 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../17-python3-rbd_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:09.134 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-rbd (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:09.151 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-02-19T20:28:09.159 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../18-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-02-19T20:28:09.159 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-19T20:28:09.178 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package librgw2. 2026-02-19T20:28:09.186 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../19-librgw2_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:09.186 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking librgw2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:09.203 INFO:teuthology.orchestra.run.trial013.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-19T20:28:09.204 INFO:teuthology.orchestra.run.trial013.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-02-19T20:28:09.205 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-02-19T20:28:09.278 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-02-19T20:28:09.283 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-rgw. 2026-02-19T20:28:09.291 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../20-python3-rgw_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:09.292 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-rgw (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:09.307 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package liboath0:amd64. 2026-02-19T20:28:09.315 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../21-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-02-19T20:28:09.315 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-19T20:28:09.330 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libradosstriper1. 2026-02-19T20:28:09.339 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../22-libradosstriper1_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:09.339 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libradosstriper1 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:09.350 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:28:09.352 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-rsa (4.8-1) ... 2026-02-19T20:28:09.356 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-common. 2026-02-19T20:28:09.364 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../23-ceph-common_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:09.364 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-common (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:09.429 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-02-19T20:28:09.501 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-02-19T20:28:09.580 INFO:teuthology.orchestra.run.trial013.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-19T20:28:09.581 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-02-19T20:28:09.623 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-base. 2026-02-19T20:28:09.632 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../24-ceph-base_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:09.637 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-base (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:09.661 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-19T20:28:09.662 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-19T20:28:09.694 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-02-19T20:28:09.703 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../25-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-02-19T20:28:09.703 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-02-19T20:28:09.717 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-cheroot. 2026-02-19T20:28:09.725 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../26-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-02-19T20:28:09.726 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-19T20:28:09.738 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-19T20:28:09.742 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-02-19T20:28:09.749 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../27-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-02-19T20:28:09.750 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-02-19T20:28:09.765 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-jaraco.text. 2026-02-19T20:28:09.773 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../28-python3-jaraco.text_3.6.0-2_all.deb ... 2026-02-19T20:28:09.774 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-02-19T20:28:09.788 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-02-19T20:28:09.796 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../29-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-02-19T20:28:09.796 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-02-19T20:28:09.810 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-tempora. 2026-02-19T20:28:09.819 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../30-python3-tempora_4.1.2-1_all.deb ... 2026-02-19T20:28:09.820 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-02-19T20:28:09.822 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-02-19T20:28:09.834 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-portend. 2026-02-19T20:28:09.843 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../31-python3-portend_3.0.0-1_all.deb ... 2026-02-19T20:28:09.843 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-02-19T20:28:09.858 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-02-19T20:28:09.866 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../32-python3-zc.lockfile_2.0-1_all.deb ... 2026-02-19T20:28:09.866 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-02-19T20:28:09.881 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-cherrypy3. 2026-02-19T20:28:09.889 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../33-python3-cherrypy3_18.6.1-4_all.deb ... 2026-02-19T20:28:09.890 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-02-19T20:28:09.896 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-19T20:28:09.897 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-02-19T20:28:09.913 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-natsort. 2026-02-19T20:28:09.921 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../34-python3-natsort_8.0.2-1_all.deb ... 2026-02-19T20:28:09.922 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-02-19T20:28:09.937 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-02-19T20:28:09.946 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../35-ceph-mgr-modules-core_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:09.946 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-mgr-modules-core (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:09.975 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-02-19T20:28:09.983 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../36-libsqlite3-mod-ceph_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:09.984 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libsqlite3-mod-ceph (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:09.998 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-mgr. 2026-02-19T20:28:10.006 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../37-ceph-mgr_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:10.007 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-mgr (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.027 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-mon. 2026-02-19T20:28:10.032 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-portend (3.0.0-1) ... 2026-02-19T20:28:10.036 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../38-ceph-mon_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:10.036 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-mon (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.100 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libfuse2:amd64. 2026-02-19T20:28:10.103 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:28:10.104 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-02-19T20:28:10.108 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../39-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-02-19T20:28:10.109 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-19T20:28:10.124 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-osd-classic. 2026-02-19T20:28:10.133 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../40-ceph-osd-classic_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:10.133 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-osd-classic (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.184 INFO:teuthology.orchestra.run.trial013.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-02-19T20:28:10.185 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-02-19T20:28:10.316 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-19T20:28:10.317 INFO:teuthology.orchestra.run.trial013.stdout:Setting up librados2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.318 INFO:teuthology.orchestra.run.trial013.stdout:Setting up librgw2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.319 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libsqlite3-mod-ceph (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.320 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-19T20:28:10.835 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libcephfs2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.836 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libradosstriper1 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.837 INFO:teuthology.orchestra.run.trial013.stdout:Setting up librbd1 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.838 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-mgr-modules-core (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.838 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-fuse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:10.914 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-19T20:28:10.915 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-19T20:28:11.329 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libcephfs-dev (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:11.330 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-rados (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:11.330 INFO:teuthology.orchestra.run.trial013.stdout:Setting up libcephfs-daemon (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:11.331 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-rbd (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:11.332 INFO:teuthology.orchestra.run.trial013.stdout:Setting up rbd-fuse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:11.333 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-rgw (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:11.334 INFO:teuthology.orchestra.run.trial013.stdout:Setting up python3-cephfs (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:11.335 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-common (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:11.363 INFO:teuthology.orchestra.run.trial013.stdout:Adding group ceph....done 2026-02-19T20:28:11.398 INFO:teuthology.orchestra.run.trial013.stdout:Adding system user ceph....done 2026-02-19T20:28:11.407 INFO:teuthology.orchestra.run.trial013.stdout:Setting system user ceph properties....done 2026-02-19T20:28:11.413 INFO:teuthology.orchestra.run.trial013.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-02-19T20:28:11.477 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-02-19T20:28:11.761 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-02-19T20:28:12.237 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-test (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:12.246 INFO:teuthology.orchestra.run.trial013.stdout:Setting up radosgw (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:12.501 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-19T20:28:12.501 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-19T20:28:12.869 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-base (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:12.986 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-02-19T20:28:13.386 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-mds (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:13.446 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-19T20:28:13.446 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-19T20:28:13.833 INFO:teuthology.orchestra.run.trial013.stdout:Setting up rbd-mirror (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:13.910 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-19T20:28:13.910 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-19T20:28:14.046 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-osd. 2026-02-19T20:28:14.059 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../41-ceph-osd_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:14.059 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-osd (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.075 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph. 2026-02-19T20:28:14.088 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../42-ceph_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:14.089 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.105 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-fuse. 2026-02-19T20:28:14.118 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../43-ceph-fuse_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:14.118 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-fuse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.143 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-mds. 2026-02-19T20:28:14.153 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../44-ceph-mds_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:14.154 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-mds (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.191 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package cephadm. 2026-02-19T20:28:14.200 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../45-cephadm_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:14.200 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking cephadm (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.217 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-asyncssh. 2026-02-19T20:28:14.225 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../46-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-02-19T20:28:14.226 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-19T20:28:14.247 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-02-19T20:28:14.255 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../47-ceph-mgr-cephadm_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:14.255 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-mgr-cephadm (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.276 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-repoze.lru. 2026-02-19T20:28:14.282 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-mgr (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.284 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../48-python3-repoze.lru_0.7-2_all.deb ... 2026-02-19T20:28:14.284 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-02-19T20:28:14.300 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-routes. 2026-02-19T20:28:14.309 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../49-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-02-19T20:28:14.309 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-02-19T20:28:14.329 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-02-19T20:28:14.337 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../50-ceph-mgr-dashboard_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:14.337 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-mgr-dashboard (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.362 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-19T20:28:14.362 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-19T20:28:14.737 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-mgr-k8sevents (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.747 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-osd-classic (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.763 INFO:teuthology.orchestra.run.trial013.stdout:update-alternatives: using /usr/bin/ceph-osd-classic to provide /usr/bin/ceph-osd (ceph-osd) in auto mode 2026-02-19T20:28:14.764 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-mgr-diskprediction-local (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.775 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-mon (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:14.848 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-19T20:28:14.848 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-19T20:28:15.019 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-02-19T20:28:15.029 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../51-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-02-19T20:28:15.030 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-19T20:28:15.067 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-joblib. 2026-02-19T20:28:15.076 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../52-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-02-19T20:28:15.077 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-02-19T20:28:15.104 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-02-19T20:28:15.112 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../53-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-02-19T20:28:15.113 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-02-19T20:28:15.128 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-sklearn. 2026-02-19T20:28:15.136 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../54-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-02-19T20:28:15.137 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-19T20:28:15.217 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-mgr-cephadm (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:15.218 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-02-19T20:28:15.227 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../55-ceph-mgr-diskprediction-local_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:15.228 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-mgr-diskprediction-local (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:15.229 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-mgr-dashboard (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:15.241 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-osd (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:15.329 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-19T20:28:15.329 INFO:teuthology.orchestra.run.trial013.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-19T20:28:15.380 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-cachetools. 2026-02-19T20:28:15.389 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../56-python3-cachetools_5.0.0-1_all.deb ... 2026-02-19T20:28:15.389 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-02-19T20:28:15.404 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-rsa. 2026-02-19T20:28:15.412 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../57-python3-rsa_4.8-1_all.deb ... 2026-02-19T20:28:15.413 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-rsa (4.8-1) ... 2026-02-19T20:28:15.430 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-google-auth. 2026-02-19T20:28:15.439 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../58-python3-google-auth_1.5.1-3_all.deb ... 2026-02-19T20:28:15.439 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-02-19T20:28:15.456 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-02-19T20:28:15.465 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../59-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-02-19T20:28:15.465 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-19T20:28:15.481 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-websocket. 2026-02-19T20:28:15.489 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../60-python3-websocket_1.2.3-1_all.deb ... 2026-02-19T20:28:15.489 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-02-19T20:28:15.507 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-kubernetes. 2026-02-19T20:28:15.516 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../61-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-02-19T20:28:15.516 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-19T20:28:15.617 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-02-19T20:28:15.626 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../62-ceph-mgr-k8sevents_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:15.626 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-mgr-k8sevents (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:15.640 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libonig5:amd64. 2026-02-19T20:28:15.649 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../63-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-02-19T20:28:15.649 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-19T20:28:15.665 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libjq1:amd64. 2026-02-19T20:28:15.674 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../64-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-19T20:28:15.674 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-19T20:28:15.689 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package jq. 2026-02-19T20:28:15.698 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../65-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-19T20:28:15.698 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-02-19T20:28:15.713 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package socat. 2026-02-19T20:28:15.720 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:15.721 INFO:teuthology.orchestra.run.trial013.stdout:Setting up ceph-volume (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:15.721 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../66-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-02-19T20:28:15.722 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-02-19T20:28:15.741 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package xmlstarlet. 2026-02-19T20:28:15.750 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../67-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-02-19T20:28:15.750 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-02-19T20:28:15.785 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-test. 2026-02-19T20:28:15.794 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../68-ceph-test_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:15.794 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-test (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:15.832 INFO:teuthology.orchestra.run.trial013.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-02-19T20:28:15.890 INFO:teuthology.orchestra.run.trial013.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-02-19T20:28:16.418 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package ceph-volume. 2026-02-19T20:28:16.426 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../69-ceph-volume_20.3.0-5336-g802bf111-1jammy_all.deb ... 2026-02-19T20:28:16.427 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking ceph-volume (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:16.449 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libcephfs-daemon. 2026-02-19T20:28:16.457 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../70-libcephfs-daemon_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:16.458 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libcephfs-daemon (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:16.472 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libcephfs-proxy2. 2026-02-19T20:28:16.481 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../71-libcephfs-proxy2_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:16.482 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libcephfs-proxy2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:16.496 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package libcephfs-dev. 2026-02-19T20:28:16.505 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../72-libcephfs-dev_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:16.505 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking libcephfs-dev (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:16.522 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package nvme-cli. 2026-02-19T20:28:16.531 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../73-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-02-19T20:28:16.531 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-02-19T20:28:16.546 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:16.546 INFO:teuthology.orchestra.run.trial013.stdout:Running kernel seems to be up-to-date. 2026-02-19T20:28:16.546 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:16.546 INFO:teuthology.orchestra.run.trial013.stdout:The processor microcode seems to be up-to-date. 2026-02-19T20:28:16.547 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:16.547 INFO:teuthology.orchestra.run.trial013.stdout:No services need to be restarted. 2026-02-19T20:28:16.547 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:16.547 INFO:teuthology.orchestra.run.trial013.stdout:No containers need to be restarted. 2026-02-19T20:28:16.547 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:16.547 INFO:teuthology.orchestra.run.trial013.stdout:No user sessions are running outdated binaries. 2026-02-19T20:28:16.547 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:16.547 INFO:teuthology.orchestra.run.trial013.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-19T20:28:16.563 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-02-19T20:28:16.572 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../74-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-02-19T20:28:16.572 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-19T20:28:16.601 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-iniconfig. 2026-02-19T20:28:16.609 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../75-python3-iniconfig_1.1.1-2_all.deb ... 2026-02-19T20:28:16.610 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-02-19T20:28:16.624 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-pluggy. 2026-02-19T20:28:16.632 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../76-python3-pluggy_0.13.0-7.1_all.deb ... 2026-02-19T20:28:16.633 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-02-19T20:28:16.649 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-psutil. 2026-02-19T20:28:16.657 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../77-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-02-19T20:28:16.658 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-02-19T20:28:16.676 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-py. 2026-02-19T20:28:16.684 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../78-python3-py_1.10.0-1_all.deb ... 2026-02-19T20:28:16.685 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-py (1.10.0-1) ... 2026-02-19T20:28:16.705 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-pygments. 2026-02-19T20:28:16.713 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../79-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-02-19T20:28:16.714 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-19T20:28:16.755 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-toml. 2026-02-19T20:28:16.763 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../80-python3-toml_0.10.2-1_all.deb ... 2026-02-19T20:28:16.764 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-02-19T20:28:16.779 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-pytest. 2026-02-19T20:28:16.787 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../81-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-02-19T20:28:16.787 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-02-19T20:28:16.809 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-simplejson. 2026-02-19T20:28:16.818 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../82-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-02-19T20:28:16.818 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-02-19T20:28:16.843 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package python3-webob. 2026-02-19T20:28:16.852 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../83-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-02-19T20:28:16.852 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-19T20:28:16.869 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-02-19T20:28:16.877 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../84-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-02-19T20:28:16.878 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-02-19T20:28:16.949 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package radosgw. 2026-02-19T20:28:16.958 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../85-radosgw_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:16.959 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking radosgw (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:17.119 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package rbd-fuse. 2026-02-19T20:28:17.128 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../86-rbd-fuse_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:17.129 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking rbd-fuse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:17.144 INFO:teuthology.orchestra.run.trial151.stdout:Selecting previously unselected package rbd-mirror. 2026-02-19T20:28:17.153 INFO:teuthology.orchestra.run.trial151.stdout:Preparing to unpack .../87-rbd-mirror_20.3.0-5336-g802bf111-1jammy_amd64.deb ... 2026-02-19T20:28:17.153 INFO:teuthology.orchestra.run.trial151.stdout:Unpacking rbd-mirror (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:17.207 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-02-19T20:28:17.272 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-19T20:28:17.273 INFO:teuthology.orchestra.run.trial151.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-02-19T20:28:17.347 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-02-19T20:28:17.357 INFO:teuthology.orchestra.run.trial013.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-02-19T20:28:17.373 DEBUG:teuthology.parallel:result is None 2026-02-19T20:28:17.629 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-02-19T20:28:18.012 INFO:teuthology.orchestra.run.trial151.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-02-19T20:28:18.022 INFO:teuthology.orchestra.run.trial151.stdout:Setting up cephadm (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:18.065 INFO:teuthology.orchestra.run.trial151.stdout:Adding system user cephadm....done 2026-02-19T20:28:18.071 INFO:teuthology.orchestra.run.trial151.stdout:usermod: no changes 2026-02-19T20:28:18.077 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-02-19T20:28:18.149 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-19T20:28:18.150 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-02-19T20:28:18.220 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-02-19T20:28:18.295 INFO:teuthology.orchestra.run.trial151.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-19T20:28:18.297 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-py (1.10.0-1) ... 2026-02-19T20:28:18.392 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-02-19T20:28:18.517 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-02-19T20:28:18.592 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-02-19T20:28:18.667 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-ceph-argparse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:18.735 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-19T20:28:18.736 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-02-19T20:28:18.737 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-19T20:28:18.738 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-19T20:28:18.739 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-02-19T20:28:18.858 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-02-19T20:28:18.935 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libcephfs-proxy2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:18.936 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-02-19T20:28:19.011 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-02-19T20:28:19.097 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-19T20:28:19.346 INFO:teuthology.orchestra.run.trial151.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-02-19T20:28:19.347 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-19T20:28:19.440 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-19T20:28:19.567 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-19T20:28:19.646 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-02-19T20:28:19.718 INFO:teuthology.orchestra.run.trial151.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-02-19T20:28:19.719 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-ceph-common (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:19.808 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-19T20:28:20.306 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:28:20.310 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-toml (0.10.2-1) ... 2026-02-19T20:28:20.385 INFO:teuthology.orchestra.run.trial151.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-19T20:28:20.386 INFO:teuthology.orchestra.run.trial151.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-02-19T20:28:20.387 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-02-19T20:28:20.460 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-02-19T20:28:20.533 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:28:20.534 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-rsa (4.8-1) ... 2026-02-19T20:28:20.613 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-02-19T20:28:20.687 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-02-19T20:28:20.763 INFO:teuthology.orchestra.run.trial151.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-19T20:28:20.764 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-02-19T20:28:20.847 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-19T20:28:20.848 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-19T20:28:20.924 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-19T20:28:21.008 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-02-19T20:28:21.082 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-19T20:28:21.083 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-02-19T20:28:21.215 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-portend (3.0.0-1) ... 2026-02-19T20:28:21.286 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-19T20:28:21.287 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-02-19T20:28:21.367 INFO:teuthology.orchestra.run.trial151.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-02-19T20:28:21.368 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-02-19T20:28:21.495 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-19T20:28:21.496 INFO:teuthology.orchestra.run.trial151.stdout:Setting up librados2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:21.496 INFO:teuthology.orchestra.run.trial151.stdout:Setting up librgw2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:21.497 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libsqlite3-mod-ceph (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:21.498 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-19T20:28:22.011 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libcephfs2 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.012 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libradosstriper1 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.013 INFO:teuthology.orchestra.run.trial151.stdout:Setting up librbd1 (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.014 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-mgr-modules-core (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.015 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-fuse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.086 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-19T20:28:22.086 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-19T20:28:22.494 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libcephfs-dev (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.495 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-rados (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.496 INFO:teuthology.orchestra.run.trial151.stdout:Setting up libcephfs-daemon (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.497 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-rbd (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.498 INFO:teuthology.orchestra.run.trial151.stdout:Setting up rbd-fuse (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.499 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-rgw (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.500 INFO:teuthology.orchestra.run.trial151.stdout:Setting up python3-cephfs (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.501 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-common (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:22.527 INFO:teuthology.orchestra.run.trial151.stdout:Adding group ceph....done 2026-02-19T20:28:22.561 INFO:teuthology.orchestra.run.trial151.stdout:Adding system user ceph....done 2026-02-19T20:28:22.571 INFO:teuthology.orchestra.run.trial151.stdout:Setting system user ceph properties....done 2026-02-19T20:28:22.578 INFO:teuthology.orchestra.run.trial151.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-02-19T20:28:22.663 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-02-19T20:28:22.939 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-02-19T20:28:23.328 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-test (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:23.329 INFO:teuthology.orchestra.run.trial151.stdout:Setting up radosgw (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:23.580 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-19T20:28:23.581 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-19T20:28:23.972 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-base (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:24.087 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-02-19T20:28:24.470 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-mds (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:24.545 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-19T20:28:24.545 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-19T20:28:24.951 INFO:teuthology.orchestra.run.trial151.stdout:Setting up rbd-mirror (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:25.021 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-19T20:28:25.021 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-19T20:28:25.395 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-mgr (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:25.470 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-19T20:28:25.470 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-19T20:28:25.858 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-mgr-k8sevents (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:25.866 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-osd-classic (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:25.883 INFO:teuthology.orchestra.run.trial151.stdout:update-alternatives: using /usr/bin/ceph-osd-classic to provide /usr/bin/ceph-osd (ceph-osd) in auto mode 2026-02-19T20:28:25.884 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-mgr-diskprediction-local (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:25.894 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-mon (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:25.963 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-19T20:28:25.963 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-19T20:28:26.363 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-mgr-cephadm (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:26.372 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-mgr-dashboard (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:26.384 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-osd (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:26.468 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-19T20:28:26.468 INFO:teuthology.orchestra.run.trial151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-19T20:28:26.866 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:26.867 INFO:teuthology.orchestra.run.trial151.stdout:Setting up ceph-volume (20.3.0-5336-g802bf111-1jammy) ... 2026-02-19T20:28:26.975 INFO:teuthology.orchestra.run.trial151.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-02-19T20:28:27.056 INFO:teuthology.orchestra.run.trial151.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-02-19T20:28:27.697 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout:Running kernel seems to be up-to-date. 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout:The processor microcode seems to be up-to-date. 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout:No services need to be restarted. 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout:No containers need to be restarted. 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout:No user sessions are running outdated binaries. 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:27.698 INFO:teuthology.orchestra.run.trial151.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-19T20:28:28.510 INFO:teuthology.orchestra.run.trial151.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-02-19T20:28:28.518 DEBUG:teuthology.parallel:result is None 2026-02-19T20:28:28.519 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=802bf111cd55e57f5842153d7ba2b8b964aa818a 2026-02-19T20:28:28.685 DEBUG:teuthology.orchestra.run.trial013:> dpkg-query -W -f '${Version}' ceph 2026-02-19T20:28:28.697 INFO:teuthology.orchestra.run.trial013.stdout:20.3.0-5336-g802bf111-1jammy 2026-02-19T20:28:28.698 INFO:teuthology.packaging:The installed version of ceph is 20.3.0-5336-g802bf111-1jammy 2026-02-19T20:28:28.698 INFO:teuthology.task.install:The correct ceph version 20.3.0-5336-g802bf111-1jammy is installed. 2026-02-19T20:28:28.701 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=802bf111cd55e57f5842153d7ba2b8b964aa818a 2026-02-19T20:28:28.858 DEBUG:teuthology.orchestra.run.trial151:> dpkg-query -W -f '${Version}' ceph 2026-02-19T20:28:28.869 INFO:teuthology.orchestra.run.trial151.stdout:20.3.0-5336-g802bf111-1jammy 2026-02-19T20:28:28.869 INFO:teuthology.packaging:The installed version of ceph is 20.3.0-5336-g802bf111-1jammy 2026-02-19T20:28:28.870 INFO:teuthology.task.install:The correct ceph version 20.3.0-5336-g802bf111-1jammy is installed. 2026-02-19T20:28:28.872 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-19T20:28:28.872 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:28.872 DEBUG:teuthology.orchestra.run.trial013:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-19T20:28:28.883 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:28.883 DEBUG:teuthology.orchestra.run.trial151:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-19T20:28:28.921 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-19T20:28:28.921 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:28.921 DEBUG:teuthology.orchestra.run.trial013:> sudo dd of=/usr/bin/daemon-helper 2026-02-19T20:28:28.935 DEBUG:teuthology.orchestra.run.trial013:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-19T20:28:28.986 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:28.986 DEBUG:teuthology.orchestra.run.trial151:> sudo dd of=/usr/bin/daemon-helper 2026-02-19T20:28:28.996 DEBUG:teuthology.orchestra.run.trial151:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-19T20:28:29.048 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-19T20:28:29.049 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:29.049 DEBUG:teuthology.orchestra.run.trial013:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-19T20:28:29.058 DEBUG:teuthology.orchestra.run.trial013:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-19T20:28:29.109 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:29.110 DEBUG:teuthology.orchestra.run.trial151:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-19T20:28:29.118 DEBUG:teuthology.orchestra.run.trial151:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-19T20:28:29.168 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-19T20:28:29.168 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:29.168 DEBUG:teuthology.orchestra.run.trial013:> sudo dd of=/usr/bin/stdin-killer 2026-02-19T20:28:29.178 DEBUG:teuthology.orchestra.run.trial013:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-19T20:28:29.230 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:29.230 DEBUG:teuthology.orchestra.run.trial151:> sudo dd of=/usr/bin/stdin-killer 2026-02-19T20:28:29.240 DEBUG:teuthology.orchestra.run.trial151:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-19T20:28:29.288 INFO:teuthology.run_tasks:Running task ceph... 2026-02-19T20:28:29.402 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-19T20:28:29.402 DEBUG:teuthology.orchestra.run.trial013:> sudo chmod 777 /var/log/ceph 2026-02-19T20:28:29.403 DEBUG:teuthology.orchestra.run.trial151:> sudo chmod 777 /var/log/ceph 2026-02-19T20:28:29.411 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-19T20:28:29.412 DEBUG:teuthology.orchestra.run.trial013:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-19T20:28:29.455 DEBUG:teuthology.orchestra.run.trial151:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-19T20:28:29.463 INFO:tasks.ceph:Creating extra log directories... 2026-02-19T20:28:29.463 DEBUG:teuthology.orchestra.run.trial013:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-19T20:28:29.503 DEBUG:teuthology.orchestra.run.trial151:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-19T20:28:29.532 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-19T20:28:29.532 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-19T20:28:29.532 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645', 'branch': 'wip-yuri-testing-2026-02-18-1820', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '59645', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial', 'no_nested_subset': False, 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'wip-yuri-testing-2026-02-18-1820'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri-testing-2026-02-18-1820', 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9494, 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri-testing-2026-02-18-1820', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_802bf111cd55e57f5842153d7ba2b8b964aa818a/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a', 'targets': {'trial013.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAjNtrYticDCX9pnZhRVVnflbZnQIugXBMd8zIps+2p71TYf4S3fwnv57bFRq5XnJ9JYrboYaeEBqKxKhQY5nNI=', 'trial151.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIjmP8lkx+u8iIeShbMgEV9ajpbAMXWwo0tojC0hZTyrhgo4Ro+8UWsRhCu8C++sBWAZLI6gwDwoff55PAEivAM='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-19_15:47:56', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699'} 2026-02-19T20:28:29.533 DEBUG:teuthology.orchestra.run.trial013:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-19T20:28:29.555 DEBUG:teuthology.orchestra.run.trial151:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-19T20:28:29.560 DEBUG:teuthology.orchestra.run.trial013:> sudo install -d -m0777 -- /var/run/ceph 2026-02-19T20:28:29.603 DEBUG:teuthology.orchestra.run.trial151:> sudo install -d -m0777 -- /var/run/ceph 2026-02-19T20:28:29.610 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:29.610 DEBUG:teuthology.orchestra.run.trial013:> dd if=/scratch_devs of=/dev/stdout 2026-02-19T20:28:29.654 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-19T20:28:29.654 DEBUG:teuthology.orchestra.run.trial013:> stat /dev/vg_nvme/lv_1 2026-02-19T20:28:29.701 INFO:teuthology.orchestra.run.trial013.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-19T20:28:29.702 INFO:teuthology.orchestra.run.trial013.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-19T20:28:29.702 INFO:teuthology.orchestra.run.trial013.stdout:Device: 5h/5d Inode: 898 Links: 1 2026-02-19T20:28:29.702 INFO:teuthology.orchestra.run.trial013.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-19T20:28:29.702 INFO:teuthology.orchestra.run.trial013.stdout:Access: 2026-02-19 20:27:12.143780341 +0000 2026-02-19T20:28:29.702 INFO:teuthology.orchestra.run.trial013.stdout:Modify: 2026-02-19 20:27:11.967783333 +0000 2026-02-19T20:28:29.702 INFO:teuthology.orchestra.run.trial013.stdout:Change: 2026-02-19 20:27:11.967783333 +0000 2026-02-19T20:28:29.702 INFO:teuthology.orchestra.run.trial013.stdout: Birth: - 2026-02-19T20:28:29.702 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-19T20:28:29.750 INFO:teuthology.orchestra.run.trial013.stderr:1+0 records in 2026-02-19T20:28:29.750 INFO:teuthology.orchestra.run.trial013.stderr:1+0 records out 2026-02-19T20:28:29.750 INFO:teuthology.orchestra.run.trial013.stderr:512 bytes copied, 0.000189422 s, 2.7 MB/s 2026-02-19T20:28:29.750 DEBUG:teuthology.orchestra.run.trial013:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-19T20:28:29.798 DEBUG:teuthology.orchestra.run.trial013:> stat /dev/vg_nvme/lv_2 2026-02-19T20:28:29.845 INFO:teuthology.orchestra.run.trial013.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-19T20:28:29.845 INFO:teuthology.orchestra.run.trial013.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-19T20:28:29.845 INFO:teuthology.orchestra.run.trial013.stdout:Device: 5h/5d Inode: 923 Links: 1 2026-02-19T20:28:29.845 INFO:teuthology.orchestra.run.trial013.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-19T20:28:29.845 INFO:teuthology.orchestra.run.trial013.stdout:Access: 2026-02-19 20:27:12.503774221 +0000 2026-02-19T20:28:29.845 INFO:teuthology.orchestra.run.trial013.stdout:Modify: 2026-02-19 20:27:12.319777348 +0000 2026-02-19T20:28:29.845 INFO:teuthology.orchestra.run.trial013.stdout:Change: 2026-02-19 20:27:12.319777348 +0000 2026-02-19T20:28:29.845 INFO:teuthology.orchestra.run.trial013.stdout: Birth: - 2026-02-19T20:28:29.845 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-19T20:28:29.892 INFO:teuthology.orchestra.run.trial013.stderr:1+0 records in 2026-02-19T20:28:29.892 INFO:teuthology.orchestra.run.trial013.stderr:1+0 records out 2026-02-19T20:28:29.892 INFO:teuthology.orchestra.run.trial013.stderr:512 bytes copied, 0.000178041 s, 2.9 MB/s 2026-02-19T20:28:29.893 DEBUG:teuthology.orchestra.run.trial013:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-19T20:28:29.938 DEBUG:teuthology.orchestra.run.trial013:> stat /dev/vg_nvme/lv_3 2026-02-19T20:28:29.985 INFO:teuthology.orchestra.run.trial013.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-19T20:28:29.985 INFO:teuthology.orchestra.run.trial013.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-19T20:28:29.985 INFO:teuthology.orchestra.run.trial013.stdout:Device: 5h/5d Inode: 947 Links: 1 2026-02-19T20:28:29.985 INFO:teuthology.orchestra.run.trial013.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-19T20:28:29.985 INFO:teuthology.orchestra.run.trial013.stdout:Access: 2026-02-19 20:27:12.831768644 +0000 2026-02-19T20:28:29.985 INFO:teuthology.orchestra.run.trial013.stdout:Modify: 2026-02-19 20:27:12.659771568 +0000 2026-02-19T20:28:29.985 INFO:teuthology.orchestra.run.trial013.stdout:Change: 2026-02-19 20:27:12.659771568 +0000 2026-02-19T20:28:29.985 INFO:teuthology.orchestra.run.trial013.stdout: Birth: - 2026-02-19T20:28:29.985 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-19T20:28:30.032 INFO:teuthology.orchestra.run.trial013.stderr:1+0 records in 2026-02-19T20:28:30.032 INFO:teuthology.orchestra.run.trial013.stderr:1+0 records out 2026-02-19T20:28:30.032 INFO:teuthology.orchestra.run.trial013.stderr:512 bytes copied, 0.000183571 s, 2.8 MB/s 2026-02-19T20:28:30.033 DEBUG:teuthology.orchestra.run.trial013:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-19T20:28:30.078 DEBUG:teuthology.orchestra.run.trial013:> stat /dev/vg_nvme/lv_4 2026-02-19T20:28:30.125 INFO:teuthology.orchestra.run.trial013.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-19T20:28:30.125 INFO:teuthology.orchestra.run.trial013.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-19T20:28:30.125 INFO:teuthology.orchestra.run.trial013.stdout:Device: 5h/5d Inode: 968 Links: 1 2026-02-19T20:28:30.125 INFO:teuthology.orchestra.run.trial013.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-19T20:28:30.125 INFO:teuthology.orchestra.run.trial013.stdout:Access: 2026-02-19 20:27:12.991765924 +0000 2026-02-19T20:28:30.125 INFO:teuthology.orchestra.run.trial013.stdout:Modify: 2026-02-19 20:27:12.987765991 +0000 2026-02-19T20:28:30.125 INFO:teuthology.orchestra.run.trial013.stdout:Change: 2026-02-19 20:27:12.987765991 +0000 2026-02-19T20:28:30.125 INFO:teuthology.orchestra.run.trial013.stdout: Birth: - 2026-02-19T20:28:30.125 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-19T20:28:30.173 INFO:teuthology.orchestra.run.trial013.stderr:1+0 records in 2026-02-19T20:28:30.174 INFO:teuthology.orchestra.run.trial013.stderr:1+0 records out 2026-02-19T20:28:30.174 INFO:teuthology.orchestra.run.trial013.stderr:512 bytes copied, 0.000173943 s, 2.9 MB/s 2026-02-19T20:28:30.174 DEBUG:teuthology.orchestra.run.trial013:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-19T20:28:30.219 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-19T20:28:30.219 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial013.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-19T20:28:30.219 INFO:tasks.ceph:Generating config... 2026-02-19T20:28:30.220 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-19T20:28:30.220 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-19T20:28:30.220 INFO:tasks.ceph:[client] log to stderr = False 2026-02-19T20:28:30.220 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-19T20:28:30.221 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-02-19T20:28:30.221 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.221 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.221 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.221 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.221 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.221 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.221 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.221 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.221 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-19T20:28:30.221 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-19T20:28:30.221 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-19T20:28:30.221 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-19T20:28:30.221 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-19T20:28:30.221 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-19T20:28:30.221 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-19T20:28:30.221 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-19T20:28:30.221 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-19T20:28:30.221 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-19T20:28:30.221 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-19T20:28:30.278 INFO:teuthology.orchestra.run.trial013.stdout:creating /etc/ceph/cluster1.keyring 2026-02-19T20:28:30.280 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-19T20:28:30.340 DEBUG:teuthology.orchestra.run.trial013:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-19T20:28:30.389 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.13')] 2026-02-19T20:28:30.389 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.13', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-19T20:28:30.389 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:30.389 DEBUG:teuthology.orchestra.run.trial013:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-19T20:28:30.433 DEBUG:teuthology.orchestra.run.trial013:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.13 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:monmaptool: generated fsid ce984b27-e607-434f-a544-5b69d6ba521e 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:setting min_mon_release = tentacle 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:epoch 0 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:fsid ce984b27-e607-434f-a544-5b69d6ba521e 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:last_changed 2026-02-19T20:28:30.492194+0000 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:created 2026-02-19T20:28:30.492194+0000 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:min_mon_release 20 (tentacle) 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:election_strategy: 1 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:0: [v2:10.20.193.13:3300/0,v1:10.20.193.13:6789/0] mon.a 2026-02-19T20:28:30.490 INFO:teuthology.orchestra.run.trial013.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-19T20:28:30.491 DEBUG:teuthology.orchestra.run.trial013:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-19T20:28:30.536 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID ce984b27-e607-434f-a544-5b69d6ba521e... 2026-02-19T20:28:30.537 DEBUG:teuthology.orchestra.run.trial013:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-19T20:28:30.578 DEBUG:teuthology.orchestra.run.trial151:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout:[global] 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: chdir = "" 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: auth supported = cephx 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: filestore xattr use omap = true 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: mon clock drift allowed = 1.000 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: osd crush chooseleaf type = 0 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: auth debug = true 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: ms die on old message = true 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: ms die on bug = true 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: mon pg warn max object skew = 0 2026-02-19T20:28:30.594 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: # disable pg_autoscaler by default for new pools 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd pool default size = 2 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon osd allow primary affinity = true 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon osd allow pg remap = true 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on legacy crush tunables = false 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on crush straw calc version zero = false 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on no sortbitwise = false 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on osd down out interval zero = false 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on too few osds = false 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon_allow_pool_size_one = true 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd default data pool replay window = 5 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon allow pool delete = true 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon cluster log file level = debug 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: debug asserts on shutdown = true 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon health detail to clog = false 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon host = 10.20.193.13 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon client directed command retry = 5 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on pool no app = False 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: ms inject socket failures = 5000 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: fsid = ce984b27-e607-434f-a544-5b69d6ba521e 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout:[osd] 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd journal size = 100 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd scrub load threshold = 5.0 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd scrub max interval = 600 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd mclock profile = high_recovery_ops 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd mclock skip benchmark = true 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd recover clone overlap = true 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd recovery max chunk = 1048576 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd debug shutdown = true 2026-02-19T20:28:30.595 INFO:teuthology.orchestra.run.trial013.stdout: osd debug op order = true 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: osd debug verify stray on activate = true 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: osd debug trim objects = true 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: osd open classes on start = true 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: osd debug pg log writeout = true 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: osd deep scrub update digest min age = 30 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: osd map max advance = 10 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: journal zero on create = true 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: filestore ondisk finisher threads = 3 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: filestore apply finisher threads = 3 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: bdev debug aio = true 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: osd debug misdirected ops = true 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: bluefs allocator = hybrid 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: bluestore allocator = hybrid 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: bluestore block size = 96636764160 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: bluestore fsck on mount = True 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug bluefs = 1/20 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug bluestore = 1/20 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug ms = 1 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug osd = 20 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug rocksdb = 4/10 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: mon osd backfillfull_ratio = 0.85 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: mon osd full ratio = 0.9 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: mon osd nearfull ratio = 0.8 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: osd failsafe full ratio = 0.95 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: osd objectstore = bluestore 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout:[mgr] 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug ms = 1 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug mgr = 20 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug mon = 20 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug auth = 20 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: mon reweight min pgs per osd = 4 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: mon reweight min bytes per osd = 10 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: mgr/telemetry/nag = false 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout:[mon] 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug ms = 1 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug mon = 20 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug paxos = 20 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: debug auth = 20 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: mon data avail warn = 5 2026-02-19T20:28:30.596 INFO:teuthology.orchestra.run.trial013.stdout: mon mgr mkfs grace = 240 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: mon reweight min pgs per osd = 4 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: mon osd reporter subtree level = osd 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: mon osd prime pg temp = true 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: mon reweight min bytes per osd = 10 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: auth mon ticket ttl = 660 # 11m 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: auth service ticket ttl = 240 # 4m 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: # don't complain about insecure global_id in the test suite 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: # 1m isn't quite enough 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: mon_down_mkfs_grace = 2m 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: mon_warn_on_filestore_osds = false 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout:[client] 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: rgw cache enabled = true 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: rgw enable ops log = true 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: rgw enable usage log = true 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: debug rbd = 20 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: debug rbd_mirror = 15 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: log to stderr = False 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: rbd default features = 125 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: rbd mirror image policy type = simple 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout:[mon.a] 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.0] 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.1] 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.2] 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.3] 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.597 INFO:teuthology.orchestra.run.trial013.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout:[global] 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: chdir = "" 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: auth supported = cephx 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: filestore xattr use omap = true 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: mon clock drift allowed = 1.000 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: osd crush chooseleaf type = 0 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: auth debug = true 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: ms die on old message = true 2026-02-19T20:28:30.599 INFO:teuthology.orchestra.run.trial151.stdout: ms die on bug = true 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon pg warn max object skew = 0 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: # disable pg_autoscaler by default for new pools 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd pool default size = 2 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon osd allow primary affinity = true 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon osd allow pg remap = true 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on legacy crush tunables = false 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on crush straw calc version zero = false 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on no sortbitwise = false 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on osd down out interval zero = false 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on too few osds = false 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon_allow_pool_size_one = true 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd default data pool replay window = 5 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon allow pool delete = true 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon cluster log file level = debug 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: debug asserts on shutdown = true 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon health detail to clog = false 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon host = 10.20.193.13 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon client directed command retry = 5 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on pool no app = False 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: ms inject socket failures = 5000 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: fsid = ce984b27-e607-434f-a544-5b69d6ba521e 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout:[osd] 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd journal size = 100 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd scrub load threshold = 5.0 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd scrub max interval = 600 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd mclock profile = high_recovery_ops 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd mclock skip benchmark = true 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd recover clone overlap = true 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: osd recovery max chunk = 1048576 2026-02-19T20:28:30.600 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd debug shutdown = true 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd debug op order = true 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd debug verify stray on activate = true 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd debug trim objects = true 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd open classes on start = true 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd debug pg log writeout = true 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd deep scrub update digest min age = 30 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd map max advance = 10 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: journal zero on create = true 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: filestore ondisk finisher threads = 3 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: filestore apply finisher threads = 3 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: bdev debug aio = true 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd debug misdirected ops = true 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: bluefs allocator = hybrid 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: bluestore allocator = hybrid 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: bluestore block size = 96636764160 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: bluestore fsck on mount = True 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug bluefs = 1/20 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug bluestore = 1/20 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug ms = 1 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug osd = 20 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug rocksdb = 4/10 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: mon osd backfillfull_ratio = 0.85 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: mon osd full ratio = 0.9 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: mon osd nearfull ratio = 0.8 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd failsafe full ratio = 0.95 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: osd objectstore = bluestore 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout:[mgr] 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug ms = 1 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug mgr = 20 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug mon = 20 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug auth = 20 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: mon reweight min pgs per osd = 4 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: mon reweight min bytes per osd = 10 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: mgr/telemetry/nag = false 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout:[mon] 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug ms = 1 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug mon = 20 2026-02-19T20:28:30.601 INFO:teuthology.orchestra.run.trial151.stdout: debug paxos = 20 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: debug auth = 20 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: mon data avail warn = 5 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: mon mgr mkfs grace = 240 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: mon reweight min pgs per osd = 4 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: mon osd reporter subtree level = osd 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: mon osd prime pg temp = true 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: mon reweight min bytes per osd = 10 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: auth mon ticket ttl = 660 # 11m 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: auth service ticket ttl = 240 # 4m 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: # don't complain about insecure global_id in the test suite 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: # 1m isn't quite enough 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: mon_down_mkfs_grace = 2m 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: mon_warn_on_filestore_osds = false 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout:[client] 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: rgw cache enabled = true 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: rgw enable ops log = true 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: rgw enable usage log = true 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: debug rbd = 20 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: debug rbd_mirror = 15 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: log to stderr = False 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: rbd default features = 125 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: rbd mirror image policy type = simple 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout:[mon.a] 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.0] 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.1] 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.2] 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.3] 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:30.602 INFO:teuthology.orchestra.run.trial151.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:30.607 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-19T20:28:30.607 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-19T20:28:30.667 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-19T20:28:30.668 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:30.668 DEBUG:teuthology.orchestra.run.trial013:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-19T20:28:30.713 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:30.713 DEBUG:teuthology.orchestra.run.trial013:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-19T20:28:30.757 INFO:tasks.ceph:Sending monmap to node ubuntu@trial013.front.sepia.ceph.com 2026-02-19T20:28:30.757 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:30.757 DEBUG:teuthology.orchestra.run.trial013:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-19T20:28:30.757 DEBUG:teuthology.orchestra.run.trial013:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-19T20:28:30.811 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:30.812 DEBUG:teuthology.orchestra.run.trial013:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-19T20:28:30.858 INFO:tasks.ceph:Sending monmap to node ubuntu@trial151.front.sepia.ceph.com 2026-02-19T20:28:30.858 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:30.858 DEBUG:teuthology.orchestra.run.trial151:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-19T20:28:30.858 DEBUG:teuthology.orchestra.run.trial151:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-19T20:28:30.874 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:30.874 DEBUG:teuthology.orchestra.run.trial151:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-19T20:28:30.920 INFO:tasks.ceph:Setting up mon nodes... 2026-02-19T20:28:30.921 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-19T20:28:30.921 DEBUG:teuthology.orchestra.run.trial013:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-19T20:28:30.943 INFO:teuthology.orchestra.run.trial013.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-19T20:28:30.946 INFO:tasks.ceph:Setting up mds nodes... 2026-02-19T20:28:30.946 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-19T20:28:30.947 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-19T20:28:31.013 INFO:teuthology.orchestra.run.trial013.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-19T20:28:31.021 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-19T20:28:31.047 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-19T20:28:31.057 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-19T20:28:31.124 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-19T20:28:31.131 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-19T20:28:31.195 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-19T20:28:31.203 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-19T20:28:31.267 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-19T20:28:31.276 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-19T20:28:31.339 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-19T20:28:31.348 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-19T20:28:31.410 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-19T20:28:31.419 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-19T20:28:31.487 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-19T20:28:31.496 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-19T20:28:31.553 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-19T20:28:31.559 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-19T20:28:31.560 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial013.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-19T20:28:31.560 DEBUG:teuthology.orchestra.run.trial013:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-19T20:28:31.569 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-19T20:28:31.570 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-19T20:28:31.570 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial013.front.sepia.ceph.com 2026-02-19T20:28:31.570 DEBUG:teuthology.orchestra.run.trial013:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-19T20:28:31.620 INFO:teuthology.orchestra.run.trial013.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-19T20:28:31.620 INFO:teuthology.orchestra.run.trial013.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-19T20:28:31.620 INFO:teuthology.orchestra.run.trial013.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-19T20:28:31.620 INFO:teuthology.orchestra.run.trial013.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-19T20:28:31.620 INFO:teuthology.orchestra.run.trial013.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-19T20:28:31.620 INFO:teuthology.orchestra.run.trial013.stdout: = sunit=0 swidth=0 blks 2026-02-19T20:28:31.621 INFO:teuthology.orchestra.run.trial013.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-19T20:28:31.621 INFO:teuthology.orchestra.run.trial013.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-19T20:28:31.621 INFO:teuthology.orchestra.run.trial013.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-19T20:28:31.621 INFO:teuthology.orchestra.run.trial013.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-19T20:28:32.195 INFO:teuthology.orchestra.run.trial013.stdout:Discarding blocks...Done. 2026-02-19T20:28:32.196 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial013.front.sepia.ceph.com -o noatime 2026-02-19T20:28:32.197 DEBUG:teuthology.orchestra.run.trial013:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-19T20:28:32.254 DEBUG:teuthology.orchestra.run.trial013:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-19T20:28:32.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T20:28:32.304 INFO:teuthology.orchestra.run.trial013.stderr:sudo: /sbin/restorecon: command not found 2026-02-19T20:28:32.305 DEBUG:teuthology.orchestra.run.trial013:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-19T20:28:32.353 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-19T20:28:32.353 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-19T20:28:32.353 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial013.front.sepia.ceph.com 2026-02-19T20:28:32.353 DEBUG:teuthology.orchestra.run.trial013:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-19T20:28:32.403 INFO:teuthology.orchestra.run.trial013.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-19T20:28:32.403 INFO:teuthology.orchestra.run.trial013.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-19T20:28:32.403 INFO:teuthology.orchestra.run.trial013.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-19T20:28:32.403 INFO:teuthology.orchestra.run.trial013.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-19T20:28:32.403 INFO:teuthology.orchestra.run.trial013.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-19T20:28:32.403 INFO:teuthology.orchestra.run.trial013.stdout: = sunit=0 swidth=0 blks 2026-02-19T20:28:32.403 INFO:teuthology.orchestra.run.trial013.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-19T20:28:32.403 INFO:teuthology.orchestra.run.trial013.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-19T20:28:32.403 INFO:teuthology.orchestra.run.trial013.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-19T20:28:32.403 INFO:teuthology.orchestra.run.trial013.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-19T20:28:32.968 INFO:teuthology.orchestra.run.trial013.stdout:Discarding blocks...Done. 2026-02-19T20:28:32.969 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial013.front.sepia.ceph.com -o noatime 2026-02-19T20:28:32.970 DEBUG:teuthology.orchestra.run.trial013:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-19T20:28:33.027 DEBUG:teuthology.orchestra.run.trial013:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-19T20:28:33.075 INFO:teuthology.orchestra.run.trial013.stderr:sudo: /sbin/restorecon: command not found 2026-02-19T20:28:33.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T20:28:33.075 DEBUG:teuthology.orchestra.run.trial013:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-19T20:28:33.126 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-19T20:28:33.126 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-19T20:28:33.126 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial013.front.sepia.ceph.com 2026-02-19T20:28:33.126 DEBUG:teuthology.orchestra.run.trial013:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-19T20:28:33.175 INFO:teuthology.orchestra.run.trial013.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-19T20:28:33.175 INFO:teuthology.orchestra.run.trial013.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-19T20:28:33.176 INFO:teuthology.orchestra.run.trial013.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-19T20:28:33.176 INFO:teuthology.orchestra.run.trial013.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-19T20:28:33.176 INFO:teuthology.orchestra.run.trial013.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-19T20:28:33.176 INFO:teuthology.orchestra.run.trial013.stdout: = sunit=0 swidth=0 blks 2026-02-19T20:28:33.176 INFO:teuthology.orchestra.run.trial013.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-19T20:28:33.176 INFO:teuthology.orchestra.run.trial013.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-19T20:28:33.176 INFO:teuthology.orchestra.run.trial013.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-19T20:28:33.176 INFO:teuthology.orchestra.run.trial013.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-19T20:28:33.738 INFO:teuthology.orchestra.run.trial013.stdout:Discarding blocks...Done. 2026-02-19T20:28:33.740 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial013.front.sepia.ceph.com -o noatime 2026-02-19T20:28:33.740 DEBUG:teuthology.orchestra.run.trial013:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-19T20:28:33.799 DEBUG:teuthology.orchestra.run.trial013:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-19T20:28:33.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T20:28:33.848 INFO:teuthology.orchestra.run.trial013.stderr:sudo: /sbin/restorecon: command not found 2026-02-19T20:28:33.849 DEBUG:teuthology.orchestra.run.trial013:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-19T20:28:33.911 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:33.909+0000 7f72d82d9a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-19T20:28:33.912 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:33.909+0000 7f72d82d9a40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-19T20:28:33.912 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:33.909+0000 7f72d82d9a40 -1 bdev(0x5623b5083800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-19T20:28:33.912 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:33.909+0000 7f72d82d9a40 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-19T20:28:34.860 DEBUG:teuthology.orchestra.run.trial013:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-19T20:28:34.911 DEBUG:teuthology.orchestra.run.trial013:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-19T20:28:34.975 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:34.973+0000 7f53b157ca40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-19T20:28:34.976 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:34.973+0000 7f53b157ca40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-19T20:28:34.976 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:34.973+0000 7f53b157ca40 -1 bdev(0x55b51f043800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-19T20:28:34.976 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:34.973+0000 7f53b157ca40 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-19T20:28:35.826 DEBUG:teuthology.orchestra.run.trial013:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-19T20:28:35.875 DEBUG:teuthology.orchestra.run.trial013:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-19T20:28:35.939 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:35.937+0000 7fedebebca40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-19T20:28:35.940 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:35.937+0000 7fedebebca40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-19T20:28:35.940 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:35.937+0000 7fedebebca40 -1 bdev(0x5577cc2c1800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-19T20:28:35.940 INFO:teuthology.orchestra.run.trial013.stderr:2026-02-19T20:28:35.937+0000 7fedebebca40 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-19T20:28:36.811 DEBUG:teuthology.orchestra.run.trial013:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-19T20:28:36.863 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-19T20:28:36.863 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:36.864 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-19T20:28:36.915 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:36.916 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-19T20:28:36.967 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:36.967 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-19T20:28:37.019 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:37.019 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-19T20:28:37.071 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:37.071 DEBUG:teuthology.orchestra.run.trial013:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-19T20:28:37.118 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:37.118 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-19T20:28:37.124 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:37.124 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-19T20:28:37.169 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:37.169 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-19T20:28:37.216 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:37.217 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-19T20:28:37.261 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:37.261 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-19T20:28:37.309 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:37.309 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-19T20:28:37.358 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:37.358 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-19T20:28:37.405 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:37.405 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-19T20:28:37.452 INFO:tasks.ceph:Adding keys to all mons... 2026-02-19T20:28:37.453 DEBUG:teuthology.orchestra.run.trial013:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-19T20:28:37.461 INFO:teuthology.orchestra.run.trial013.stdout:[mgr.x] 2026-02-19T20:28:37.461 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBucpdpBVRbOBAAlxsg6ARZ/yXPOseRE0/utQ== 2026-02-19T20:28:37.461 INFO:teuthology.orchestra.run.trial013.stdout:[osd.0] 2026-02-19T20:28:37.461 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBxcpdpcMlsNhAABjtMPvqTX7Lp0nRz/SUT+g== 2026-02-19T20:28:37.461 INFO:teuthology.orchestra.run.trial013.stdout:[osd.1] 2026-02-19T20:28:37.461 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBycpdpn7g7OhAATiEG4gYRcLy3D/Xf5UY4mg== 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout:[osd.2] 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBzcpdpzLMTOBAAQvhAcV99fkDm1WUiqbWrYg== 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout:[client.0] 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBvcpdp5qvjABAAaWAbkMRoqT8GCGkXgYBAeg== 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror] 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBvcpdpwiy/AhAALN36hFPTJlLpRD7X4t1yWw== 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.0] 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBvcpdp29hHBxAA75mE77Lrx+T56ICXRBj7vg== 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.1] 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBvcpdpxZ6PCxAAaCu0jY7IYUPl8SiplorTjQ== 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.2] 2026-02-19T20:28:37.462 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBvcpdpdC/VDxAAcLmHLrLf1OLTa+cbt7DEng== 2026-02-19T20:28:37.463 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.3] 2026-02-19T20:28:37.463 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBvcpdpFFEcFBAAJDHP4h0TREmJhMsVHEHq3w== 2026-02-19T20:28:37.463 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.4] 2026-02-19T20:28:37.463 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBvcpdpXmtiGBAAHn2cpkb0Bokg8ATI3layDw== 2026-02-19T20:28:37.463 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.5] 2026-02-19T20:28:37.463 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBvcpdpp+fpHBAAOAs77EDIXsvqI2dhntQlwg== 2026-02-19T20:28:37.463 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.6] 2026-02-19T20:28:37.463 INFO:teuthology.orchestra.run.trial013.stdout: key = AQBvcpdpLDjcIBAAGNgQBcg3Twe6wOJydlQbPw== 2026-02-19T20:28:37.464 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-19T20:28:37.528 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-19T20:28:37.591 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-19T20:28:37.655 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-19T20:28:37.723 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:37.788 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:37.852 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:37.916 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:37.980 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:38.045 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:38.109 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:38.172 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:38.236 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:38.301 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-19T20:28:38.301 DEBUG:teuthology.orchestra.run.trial013:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-19T20:28:38.350 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-19T20:28:38.421 DEBUG:teuthology.orchestra.run.trial013:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-19T20:28:38.471 DEBUG:teuthology.orchestra.run.trial013:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-19T20:28:38.517 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-19T20:28:38.518 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-19T20:28:38.518 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-19T20:28:38.559 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-19T20:28:38.559 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-19T20:28:38.560 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-19T20:28:38.560 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-19T20:28:38.562 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-19T20:28:38.562 DEBUG:tasks.ceph:set 0 configs 2026-02-19T20:28:38.563 DEBUG:teuthology.orchestra.run.trial013:> sudo ceph --cluster cluster1 config dump 2026-02-19T20:28:38.679 INFO:teuthology.orchestra.run.trial013.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-19T20:28:38.691 INFO:tasks.ceph:Setting crush tunables to default 2026-02-19T20:28:38.691 DEBUG:teuthology.orchestra.run.trial013:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-19T20:28:38.785 INFO:teuthology.orchestra.run.trial013.stderr:adjusted tunables profile to default 2026-02-19T20:28:38.795 INFO:tasks.ceph:check_enable_crimson: False 2026-02-19T20:28:38.795 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-19T20:28:38.795 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:38.795 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-19T20:28:38.804 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:38.804 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-19T20:28:38.854 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:38.854 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-19T20:28:38.901 DEBUG:teuthology.orchestra.run.trial013:> sudo ceph --cluster cluster1 osd new b21c7b36-5ce8-4233-a9f0-ba75115b2697 0 2026-02-19T20:28:39.040 INFO:teuthology.orchestra.run.trial013.stdout:0 2026-02-19T20:28:39.050 DEBUG:teuthology.orchestra.run.trial013:> sudo ceph --cluster cluster1 osd new c35eb6ed-9cea-4110-8e0d-078ee3016838 1 2026-02-19T20:28:39.146 INFO:teuthology.orchestra.run.trial013.stdout:1 2026-02-19T20:28:39.156 DEBUG:teuthology.orchestra.run.trial013:> sudo ceph --cluster cluster1 osd new 16338c81-9245-466a-98d3-99bcc7dbf2b6 2 2026-02-19T20:28:39.255 INFO:teuthology.orchestra.run.trial013.stdout:2 2026-02-19T20:28:39.265 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-19T20:28:39.265 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-19T20:28:39.267 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-19T20:28:39.268 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-19T20:28:39.268 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-19T20:28:39.270 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-19T20:28:39.270 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-19T20:28:39.270 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-19T20:28:39.272 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-19T20:28:39.272 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-19T20:28:39.368 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:39.368 INFO:teuthology.orchestra.run.trial013.stdout:{"epoch":5,"fsid":"ce984b27-e607-434f-a544-5b69d6ba521e","created":"2026-02-19T20:28:38.641475+0000","modified":"2026-02-19T20:28:39.256107+0000","last_up_change":"0.000000","last_in_change":"2026-02-19T20:28:39.256107+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"b21c7b36-5ce8-4233-a9f0-ba75115b2697","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"c35eb6ed-9cea-4110-8e0d-078ee3016838","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"16338c81-9245-466a-98d3-99bcc7dbf2b6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T20:28:39.380 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-19T20:28:39.380 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-19T20:28:39.683 DEBUG:teuthology.orchestra.run.trial013:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-19T20:28:39.876 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:28:39.873+0000 7f93b9c15a40 -1 Falling back to public interface 2026-02-19T20:28:39.907 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:28:39.905+0000 7fe4ee5f4a40 -1 Falling back to public interface 2026-02-19T20:28:39.908 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:28:39.905+0000 7fd64bd7aa40 -1 Falling back to public interface 2026-02-19T20:28:39.943 INFO:teuthology.misc.health.trial013.stderr:2026-02-19T20:28:39.941+0000 7fcb677fe640 0 --2- 10.20.193.13:0/1414709086 >> v2:10.20.193.13:3300/0 conn(0x7fcb680a24a0 0x7fcb6805b8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 gs=4 cs=0 l=0 c_cookie=0 s_cookie=0 reconnecting=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-19T20:28:39.976 INFO:teuthology.misc.health.trial013.stdout: 2026-02-19T20:28:39.976 INFO:teuthology.misc.health.trial013.stdout:{"epoch":5,"fsid":"ce984b27-e607-434f-a544-5b69d6ba521e","created":"2026-02-19T20:28:38.641475+0000","modified":"2026-02-19T20:28:39.256107+0000","last_up_change":"0.000000","last_in_change":"2026-02-19T20:28:39.256107+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"b21c7b36-5ce8-4233-a9f0-ba75115b2697","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"c35eb6ed-9cea-4110-8e0d-078ee3016838","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"16338c81-9245-466a-98d3-99bcc7dbf2b6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T20:28:39.987 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-19T20:28:40.228 INFO:tasks.ceph.cluster1.mgr.x.trial013.stderr:ERROR:root:Module 'jmespath' is not installed. 2026-02-19T20:28:40.283 INFO:tasks.ceph.cluster1.mgr.x.trial013.stderr:ERROR:dashboard.controllers.nvmeof:Failed to import NVMeoFClient and related components: cannot import name 'NVMeoFClient' from 'dashboard.services.nvmeof_client' (/usr/share/ceph/mgr/dashboard/services/nvmeof_client.py) 2026-02-19T20:28:40.288 INFO:tasks.ceph.cluster1.mgr.x.trial013.stderr:ERROR:sso:Module 'jmespath' is not installed. 2026-02-19T20:28:40.339 INFO:tasks.ceph.cluster1.mgr.x.trial013.stderr:ERROR:root:Module 'xmltodict' is not installed. 2026-02-19T20:28:40.778 INFO:tasks.ceph.cluster1.mgr.x.trial013.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-19T20:28:40.779 INFO:tasks.ceph.cluster1.mgr.x.trial013.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-19T20:28:40.779 INFO:tasks.ceph.cluster1.mgr.x.trial013.stderr: from numpy import show_config as show_numpy_config 2026-02-19T20:28:41.270 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:28:41.265+0000 7fe4ee5f4a40 -1 osd.2 0 log_to_monitors true 2026-02-19T20:28:41.270 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:28:41.269+0000 7f93b9c15a40 -1 osd.0 0 log_to_monitors true 2026-02-19T20:28:41.349 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:28:41.345+0000 7fd64bd7aa40 -1 osd.1 0 log_to_monitors true 2026-02-19T20:28:41.855 INFO:tasks.ceph.cluster1.mgr.x.trial013.stderr:2026-02-19T20:28:41.853+0000 7f57af179640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-19T20:28:42.887 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:28:42.885+0000 7fd647d24640 -1 osd.1 0 waiting for initial osdmap 2026-02-19T20:28:42.888 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:28:42.885+0000 7f93b5bbf640 -1 osd.0 0 waiting for initial osdmap 2026-02-19T20:28:42.888 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:28:42.885+0000 7fe4ea59e640 -1 osd.2 0 waiting for initial osdmap 2026-02-19T20:28:42.890 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:28:42.885+0000 7fd63f3f3640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-19T20:28:42.890 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:28:42.885+0000 7f93ad28e640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-19T20:28:42.890 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:28:42.889+0000 7fe4e1c6d640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-19T20:28:46.290 DEBUG:teuthology.orchestra.run.trial013:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-19T20:28:46.441 INFO:teuthology.misc.health.trial013.stdout: 2026-02-19T20:28:46.441 INFO:teuthology.misc.health.trial013.stdout:{"epoch":10,"fsid":"ce984b27-e607-434f-a544-5b69d6ba521e","created":"2026-02-19T20:28:38.641475+0000","modified":"2026-02-19T20:28:45.916570+0000","last_up_change":"2026-02-19T20:28:43.886354+0000","last_in_change":"2026-02-19T20:28:39.256107+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-19T20:28:44.883525+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b21c7b36-5ce8-4233-a9f0-ba75115b2697","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6800","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6801","nonce":3970146534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6802","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6803","nonce":3970146534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6806","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6807","nonce":3970146534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6804","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6805","nonce":3970146534}]},"public_addr":"10.20.193.13:6801/3970146534","cluster_addr":"10.20.193.13:6803/3970146534","heartbeat_back_addr":"10.20.193.13:6807/3970146534","heartbeat_front_addr":"10.20.193.13:6805/3970146534","state":["exists","up"]},{"osd":1,"uuid":"c35eb6ed-9cea-4110-8e0d-078ee3016838","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6810","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6811","nonce":61555766}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6813","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6816","nonce":61555766}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6822","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6823","nonce":61555766}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6818","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6820","nonce":61555766}]},"public_addr":"10.20.193.13:6811/61555766","cluster_addr":"10.20.193.13:6816/61555766","heartbeat_back_addr":"10.20.193.13:6823/61555766","heartbeat_front_addr":"10.20.193.13:6820/61555766","state":["exists","up"]},{"osd":2,"uuid":"16338c81-9245-466a-98d3-99bcc7dbf2b6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6808","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6809","nonce":2806143231}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6812","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6814","nonce":2806143231}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6819","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6821","nonce":2806143231}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6815","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6817","nonce":2806143231}]},"public_addr":"10.20.193.13:6809/2806143231","cluster_addr":"10.20.193.13:6814/2806143231","heartbeat_back_addr":"10.20.193.13:6821/2806143231","heartbeat_front_addr":"10.20.193.13:6817/2806143231","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T20:28:46.451 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-19T20:28:46.451 INFO:tasks.ceph:Creating RBD pool 2026-02-19T20:28:46.451 DEBUG:teuthology.orchestra.run.trial013:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-19T20:28:46.916 INFO:teuthology.orchestra.run.trial013.stderr:pool 'rbd' created 2026-02-19T20:28:46.930 DEBUG:teuthology.orchestra.run.trial013:> rbd --cluster cluster1 pool init rbd 2026-02-19T20:28:49.934 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-19T20:28:49.935 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2026-02-19T20:28:49.935 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-19T20:28:50.092 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:50.102 INFO:teuthology.orchestra.run.trial013.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-19T20:28:50.102 INFO:tasks.ceph_manager:config epoch is 1 2026-02-19T20:28:50.103 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-19T20:28:50.103 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-19T20:28:50.103 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-19T20:28:50.283 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:50.293 INFO:teuthology.orchestra.run.trial013.stdout:{"epoch":5,"flags":0,"active_gid":4104,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6824","nonce":4287533568},{"type":"v1","addr":"10.20.193.13:6825","nonce":4287533568}]},"active_addr":"10.20.193.13:6825/4287533568","active_change":"2026-02-19T20:28:40.848754+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often the agent on each host will gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"The first TCP port the agent will try to bind to. Up to 1000 subsequent ports will be attempted if this port cannot be bound","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this option can facilitate debugging daemons that encounter runtime problems.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"How frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"Fraction of total system memory to divide among autotuned daemons. Converged systems hosting both Ceph and compute should set a lower value; systems with more than 10 GiB of memory per OSD may benefit from a slightly larger value so that the abundance of RAM can be fully utilized","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self-signed certificates generated and signed by the cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently Podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_alloy":{"name":"container_image_alloy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/alloy:latest","min":"","max":"","enum_allowed":[],"desc":"Alloy container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.6","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"Seconds for which to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"Seconds for which to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"Seconds for which to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"Location of dashboards to include in Grafana","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"How frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether the hardware monitoring daemon be deployed on every host?","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at the debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"Max number of daemons per service per host: guard against mistakes and runaways","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"Max number of OSDs that will be drained simultaneously when OSDs are removed","long_desc":"","tags":[],"see_also":[]},"max_parallel_osd_upgrades":{"name":"max_parallel_osd_upgrades","type":"int","level":"advanced","flags":0,"default_value":"16","min":"","max":"","enum_allowed":[],"desc":"Maximum number of OSD daemons upgraded in parallel.","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"Mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default IP address for RedFish API (OOB management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"Location of alerts to include in Prometheus Alertmanager","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry URL for login purposes. This is not the default registry.","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"Cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Customized SSH config file to distribute to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many sequential SSH connection liveness check failures before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often SSH connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"How frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use the cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest to ensure that all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"healthcheck_history_max_entries":{"name":"healthcheck_history_max_entries","type":"int","level":"advanced","flags":1,"default_value":"1000","min":"","max":"","enum_allowed":[],"desc":"Maximum number of health check history entries to keep","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.13:0","nonce":1195808183}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.13:0","nonce":4017812473}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.13:0","nonce":345943228}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.13:0","nonce":3550143595}]}]} 2026-02-19T20:28:50.296 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-19T20:28:50.297 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-19T20:28:50.297 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-19T20:28:50.455 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:50.455 INFO:teuthology.orchestra.run.trial013.stdout:{"epoch":14,"fsid":"ce984b27-e607-434f-a544-5b69d6ba521e","created":"2026-02-19T20:28:38.641475+0000","modified":"2026-02-19T20:28:49.925979+0000","last_up_change":"2026-02-19T20:28:43.886354+0000","last_in_change":"2026-02-19T20:28:39.256107+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-19T20:28:44.883525+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-19T20:28:46.611861+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b21c7b36-5ce8-4233-a9f0-ba75115b2697","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6800","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6801","nonce":3970146534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6802","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6803","nonce":3970146534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6806","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6807","nonce":3970146534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6804","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6805","nonce":3970146534}]},"public_addr":"10.20.193.13:6801/3970146534","cluster_addr":"10.20.193.13:6803/3970146534","heartbeat_back_addr":"10.20.193.13:6807/3970146534","heartbeat_front_addr":"10.20.193.13:6805/3970146534","state":["exists","up"]},{"osd":1,"uuid":"c35eb6ed-9cea-4110-8e0d-078ee3016838","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6810","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6811","nonce":61555766}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6813","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6816","nonce":61555766}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6822","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6823","nonce":61555766}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6818","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6820","nonce":61555766}]},"public_addr":"10.20.193.13:6811/61555766","cluster_addr":"10.20.193.13:6816/61555766","heartbeat_back_addr":"10.20.193.13:6823/61555766","heartbeat_front_addr":"10.20.193.13:6820/61555766","state":["exists","up"]},{"osd":2,"uuid":"16338c81-9245-466a-98d3-99bcc7dbf2b6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6808","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6809","nonce":2806143231}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6812","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6814","nonce":2806143231}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6819","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6821","nonce":2806143231}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6815","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6817","nonce":2806143231}]},"public_addr":"10.20.193.13:6809/2806143231","cluster_addr":"10.20.193.13:6814/2806143231","heartbeat_back_addr":"10.20.193.13:6821/2806143231","heartbeat_front_addr":"10.20.193.13:6817/2806143231","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T20:28:50.466 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-19T20:28:50.466 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-19T20:28:50.625 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:50.625 INFO:teuthology.orchestra.run.trial013.stdout:{"epoch":14,"fsid":"ce984b27-e607-434f-a544-5b69d6ba521e","created":"2026-02-19T20:28:38.641475+0000","modified":"2026-02-19T20:28:49.925979+0000","last_up_change":"2026-02-19T20:28:43.886354+0000","last_in_change":"2026-02-19T20:28:39.256107+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-19T20:28:44.883525+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-19T20:28:46.611861+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b21c7b36-5ce8-4233-a9f0-ba75115b2697","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6800","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6801","nonce":3970146534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6802","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6803","nonce":3970146534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6806","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6807","nonce":3970146534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6804","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6805","nonce":3970146534}]},"public_addr":"10.20.193.13:6801/3970146534","cluster_addr":"10.20.193.13:6803/3970146534","heartbeat_back_addr":"10.20.193.13:6807/3970146534","heartbeat_front_addr":"10.20.193.13:6805/3970146534","state":["exists","up"]},{"osd":1,"uuid":"c35eb6ed-9cea-4110-8e0d-078ee3016838","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6810","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6811","nonce":61555766}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6813","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6816","nonce":61555766}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6822","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6823","nonce":61555766}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6818","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6820","nonce":61555766}]},"public_addr":"10.20.193.13:6811/61555766","cluster_addr":"10.20.193.13:6816/61555766","heartbeat_back_addr":"10.20.193.13:6823/61555766","heartbeat_front_addr":"10.20.193.13:6820/61555766","state":["exists","up"]},{"osd":2,"uuid":"16338c81-9245-466a-98d3-99bcc7dbf2b6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6808","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6809","nonce":2806143231}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6812","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6814","nonce":2806143231}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6819","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6821","nonce":2806143231}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6815","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6817","nonce":2806143231}]},"public_addr":"10.20.193.13:6809/2806143231","cluster_addr":"10.20.193.13:6814/2806143231","heartbeat_back_addr":"10.20.193.13:6821/2806143231","heartbeat_front_addr":"10.20.193.13:6817/2806143231","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T20:28:50.639 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-19T20:28:50.640 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-19T20:28:50.641 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-19T20:28:50.728 INFO:teuthology.orchestra.run.trial013.stdout:34359738371 2026-02-19T20:28:50.729 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-19T20:28:50.730 INFO:teuthology.orchestra.run.trial013.stdout:34359738371 2026-02-19T20:28:50.731 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-19T20:28:50.731 INFO:teuthology.orchestra.run.trial013.stdout:34359738371 2026-02-19T20:28:50.732 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-19T20:28:50.928 INFO:teuthology.orchestra.run.trial013.stdout:34359738371 2026-02-19T20:28:50.930 INFO:teuthology.orchestra.run.trial013.stdout:34359738371 2026-02-19T20:28:50.934 INFO:teuthology.orchestra.run.trial013.stdout:34359738371 2026-02-19T20:28:50.940 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.1 2026-02-19T20:28:50.940 DEBUG:teuthology.parallel:result is None 2026-02-19T20:28:50.943 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.2 2026-02-19T20:28:50.943 DEBUG:teuthology.parallel:result is None 2026-02-19T20:28:50.947 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.0 2026-02-19T20:28:50.948 DEBUG:teuthology.parallel:result is None 2026-02-19T20:28:50.948 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-19T20:28:50.948 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-19T20:28:51.145 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:51.146 INFO:teuthology.orchestra.run.trial013.stderr:dumped all 2026-02-19T20:28:51.156 INFO:teuthology.orchestra.run.trial013.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-19T20:28:50.852774+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":9,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82400,"kb_used_data":1520,"kb_used_omap":23,"kb_used_meta":80808,"kb_avail":283033120,"statfs":{"total":289910292480,"available":289825914880,"internally_reserved":0,"allocated":1556480,"data_stored":1283458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23725,"internal_metadata":82748243},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"1.924495"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.932299+0000","last_change":"2026-02-19T20:28:49.932388+0000","last_active":"2026-02-19T20:28:49.932299+0000","last_peered":"2026-02-19T20:28:49.932299+0000","last_clean":"2026-02-19T20:28:49.932299+0000","last_became_active":"2026-02-19T20:28:47.929308+0000","last_became_peered":"2026-02-19T20:28:47.929308+0000","last_unstale":"2026-02-19T20:28:49.932299+0000","last_undegraded":"2026-02-19T20:28:49.932299+0000","last_fullsized":"2026-02-19T20:28:49.932299+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:51:59.644924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086370499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.931857+0000","last_change":"2026-02-19T20:28:49.932057+0000","last_active":"2026-02-19T20:28:49.931857+0000","last_peered":"2026-02-19T20:28:49.931857+0000","last_clean":"2026-02-19T20:28:49.931857+0000","last_became_active":"2026-02-19T20:28:47.929421+0000","last_became_peered":"2026-02-19T20:28:47.929421+0000","last_unstale":"2026-02-19T20:28:49.931857+0000","last_undegraded":"2026-02-19T20:28:49.931857+0000","last_fullsized":"2026-02-19T20:28:49.931857+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:53:36.261502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055284699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.932392+0000","last_change":"2026-02-19T20:28:49.932501+0000","last_active":"2026-02-19T20:28:49.932392+0000","last_peered":"2026-02-19T20:28:49.932392+0000","last_clean":"2026-02-19T20:28:49.932392+0000","last_became_active":"2026-02-19T20:28:47.929096+0000","last_became_peered":"2026-02-19T20:28:47.929096+0000","last_unstale":"2026-02-19T20:28:49.932392+0000","last_undegraded":"2026-02-19T20:28:49.932392+0000","last_fullsized":"2026-02-19T20:28:49.932392+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:35:27.823970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005578,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.932246+0000","last_change":"2026-02-19T20:28:49.932359+0000","last_active":"2026-02-19T20:28:49.932246+0000","last_peered":"2026-02-19T20:28:49.932246+0000","last_clean":"2026-02-19T20:28:49.932246+0000","last_became_active":"2026-02-19T20:28:47.929286+0000","last_became_peered":"2026-02-19T20:28:47.929286+0000","last_unstale":"2026-02-19T20:28:49.932246+0000","last_undegraded":"2026-02-19T20:28:49.932246+0000","last_fullsized":"2026-02-19T20:28:49.932246+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:15:05.977395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084824599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.930433+0000","last_change":"2026-02-19T20:28:49.930433+0000","last_active":"2026-02-19T20:28:49.930433+0000","last_peered":"2026-02-19T20:28:49.930433+0000","last_clean":"2026-02-19T20:28:49.930433+0000","last_became_active":"2026-02-19T20:28:47.929119+0000","last_became_peered":"2026-02-19T20:28:47.929119+0000","last_unstale":"2026-02-19T20:28:49.930433+0000","last_undegraded":"2026-02-19T20:28:49.930433+0000","last_fullsized":"2026-02-19T20:28:49.930433+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:02:22.704407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092100199999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:50.333424+0000","last_change":"2026-02-19T20:28:50.333526+0000","last_active":"2026-02-19T20:28:50.333424+0000","last_peered":"2026-02-19T20:28:50.333424+0000","last_clean":"2026-02-19T20:28:50.333424+0000","last_became_active":"2026-02-19T20:28:47.929462+0000","last_became_peered":"2026-02-19T20:28:47.929462+0000","last_unstale":"2026-02-19T20:28:50.333424+0000","last_undegraded":"2026-02-19T20:28:50.333424+0000","last_fullsized":"2026-02-19T20:28:50.333424+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:14:47.461186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024374299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:50.333400+0000","last_change":"2026-02-19T20:28:50.333516+0000","last_active":"2026-02-19T20:28:50.333400+0000","last_peered":"2026-02-19T20:28:50.333400+0000","last_clean":"2026-02-19T20:28:50.333400+0000","last_became_active":"2026-02-19T20:28:47.929487+0000","last_became_peered":"2026-02-19T20:28:47.929487+0000","last_unstale":"2026-02-19T20:28:50.333400+0000","last_undegraded":"2026-02-19T20:28:50.333400+0000","last_fullsized":"2026-02-19T20:28:50.333400+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T03:04:05.217623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024657800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.931821+0000","last_change":"2026-02-19T20:28:49.932091+0000","last_active":"2026-02-19T20:28:49.931821+0000","last_peered":"2026-02-19T20:28:49.931821+0000","last_clean":"2026-02-19T20:28:49.931821+0000","last_became_active":"2026-02-19T20:28:47.928693+0000","last_became_peered":"2026-02-19T20:28:47.928693+0000","last_unstale":"2026-02-19T20:28:49.931821+0000","last_undegraded":"2026-02-19T20:28:49.931821+0000","last_fullsized":"2026-02-19T20:28:49.931821+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:23:54.434881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060120699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.931158+0000","last_change":"2026-02-19T20:28:45.921802+0000","last_active":"2026-02-19T20:28:49.931158+0000","last_peered":"2026-02-19T20:28:49.931158+0000","last_clean":"2026-02-19T20:28:49.931158+0000","last_became_active":"2026-02-19T20:28:45.920826+0000","last_became_peered":"2026-02-19T20:28:45.920826+0000","last_unstale":"2026-02-19T20:28:49.931158+0000","last_undegraded":"2026-02-19T20:28:49.931158+0000","last_fullsized":"2026-02-19T20:28:49.931158+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_clean_scrub_stamp":"2026-02-19T20:28:44.914354+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:32:41.646846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27072,"kb_used_data":112,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344768,"statfs":{"total":96636764160,"available":96609042432,"internally_reserved":0,"allocated":114688,"data_stored":31852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27664,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344176,"statfs":{"total":96636764160,"available":96608436224,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27664,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344176,"statfs":{"total":96636764160,"available":96608436224,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T20:28:51.157 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-19T20:28:51.320 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:51.321 INFO:teuthology.orchestra.run.trial013.stderr:dumped all 2026-02-19T20:28:51.331 INFO:teuthology.orchestra.run.trial013.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-19T20:28:50.852774+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":9,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82400,"kb_used_data":1520,"kb_used_omap":23,"kb_used_meta":80808,"kb_avail":283033120,"statfs":{"total":289910292480,"available":289825914880,"internally_reserved":0,"allocated":1556480,"data_stored":1283458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23725,"internal_metadata":82748243},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"1.924495"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.932299+0000","last_change":"2026-02-19T20:28:49.932388+0000","last_active":"2026-02-19T20:28:49.932299+0000","last_peered":"2026-02-19T20:28:49.932299+0000","last_clean":"2026-02-19T20:28:49.932299+0000","last_became_active":"2026-02-19T20:28:47.929308+0000","last_became_peered":"2026-02-19T20:28:47.929308+0000","last_unstale":"2026-02-19T20:28:49.932299+0000","last_undegraded":"2026-02-19T20:28:49.932299+0000","last_fullsized":"2026-02-19T20:28:49.932299+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:51:59.644924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086370499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.931857+0000","last_change":"2026-02-19T20:28:49.932057+0000","last_active":"2026-02-19T20:28:49.931857+0000","last_peered":"2026-02-19T20:28:49.931857+0000","last_clean":"2026-02-19T20:28:49.931857+0000","last_became_active":"2026-02-19T20:28:47.929421+0000","last_became_peered":"2026-02-19T20:28:47.929421+0000","last_unstale":"2026-02-19T20:28:49.931857+0000","last_undegraded":"2026-02-19T20:28:49.931857+0000","last_fullsized":"2026-02-19T20:28:49.931857+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:53:36.261502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055284699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.932392+0000","last_change":"2026-02-19T20:28:49.932501+0000","last_active":"2026-02-19T20:28:49.932392+0000","last_peered":"2026-02-19T20:28:49.932392+0000","last_clean":"2026-02-19T20:28:49.932392+0000","last_became_active":"2026-02-19T20:28:47.929096+0000","last_became_peered":"2026-02-19T20:28:47.929096+0000","last_unstale":"2026-02-19T20:28:49.932392+0000","last_undegraded":"2026-02-19T20:28:49.932392+0000","last_fullsized":"2026-02-19T20:28:49.932392+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:35:27.823970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005578,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.932246+0000","last_change":"2026-02-19T20:28:49.932359+0000","last_active":"2026-02-19T20:28:49.932246+0000","last_peered":"2026-02-19T20:28:49.932246+0000","last_clean":"2026-02-19T20:28:49.932246+0000","last_became_active":"2026-02-19T20:28:47.929286+0000","last_became_peered":"2026-02-19T20:28:47.929286+0000","last_unstale":"2026-02-19T20:28:49.932246+0000","last_undegraded":"2026-02-19T20:28:49.932246+0000","last_fullsized":"2026-02-19T20:28:49.932246+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:15:05.977395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084824599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.930433+0000","last_change":"2026-02-19T20:28:49.930433+0000","last_active":"2026-02-19T20:28:49.930433+0000","last_peered":"2026-02-19T20:28:49.930433+0000","last_clean":"2026-02-19T20:28:49.930433+0000","last_became_active":"2026-02-19T20:28:47.929119+0000","last_became_peered":"2026-02-19T20:28:47.929119+0000","last_unstale":"2026-02-19T20:28:49.930433+0000","last_undegraded":"2026-02-19T20:28:49.930433+0000","last_fullsized":"2026-02-19T20:28:49.930433+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:02:22.704407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092100199999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:50.333424+0000","last_change":"2026-02-19T20:28:50.333526+0000","last_active":"2026-02-19T20:28:50.333424+0000","last_peered":"2026-02-19T20:28:50.333424+0000","last_clean":"2026-02-19T20:28:50.333424+0000","last_became_active":"2026-02-19T20:28:47.929462+0000","last_became_peered":"2026-02-19T20:28:47.929462+0000","last_unstale":"2026-02-19T20:28:50.333424+0000","last_undegraded":"2026-02-19T20:28:50.333424+0000","last_fullsized":"2026-02-19T20:28:50.333424+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:14:47.461186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024374299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:50.333400+0000","last_change":"2026-02-19T20:28:50.333516+0000","last_active":"2026-02-19T20:28:50.333400+0000","last_peered":"2026-02-19T20:28:50.333400+0000","last_clean":"2026-02-19T20:28:50.333400+0000","last_became_active":"2026-02-19T20:28:47.929487+0000","last_became_peered":"2026-02-19T20:28:47.929487+0000","last_unstale":"2026-02-19T20:28:50.333400+0000","last_undegraded":"2026-02-19T20:28:50.333400+0000","last_fullsized":"2026-02-19T20:28:50.333400+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T03:04:05.217623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024657800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.931821+0000","last_change":"2026-02-19T20:28:49.932091+0000","last_active":"2026-02-19T20:28:49.931821+0000","last_peered":"2026-02-19T20:28:49.931821+0000","last_clean":"2026-02-19T20:28:49.931821+0000","last_became_active":"2026-02-19T20:28:47.928693+0000","last_became_peered":"2026-02-19T20:28:47.928693+0000","last_unstale":"2026-02-19T20:28:49.931821+0000","last_undegraded":"2026-02-19T20:28:49.931821+0000","last_fullsized":"2026-02-19T20:28:49.931821+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:23:54.434881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060120699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:28:49.931158+0000","last_change":"2026-02-19T20:28:45.921802+0000","last_active":"2026-02-19T20:28:49.931158+0000","last_peered":"2026-02-19T20:28:49.931158+0000","last_clean":"2026-02-19T20:28:49.931158+0000","last_became_active":"2026-02-19T20:28:45.920826+0000","last_became_peered":"2026-02-19T20:28:45.920826+0000","last_unstale":"2026-02-19T20:28:49.931158+0000","last_undegraded":"2026-02-19T20:28:49.931158+0000","last_fullsized":"2026-02-19T20:28:49.931158+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_clean_scrub_stamp":"2026-02-19T20:28:44.914354+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:32:41.646846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27072,"kb_used_data":112,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344768,"statfs":{"total":96636764160,"available":96609042432,"internally_reserved":0,"allocated":114688,"data_stored":31852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27664,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344176,"statfs":{"total":96636764160,"available":96608436224,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27664,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344176,"statfs":{"total":96636764160,"available":96608436224,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T20:28:51.333 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-19T20:28:51.333 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-19T20:28:51.333 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-19T20:28:51.333 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-19T20:28:51.512 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:51.513 INFO:teuthology.orchestra.run.trial013.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-19T20:28:51.524 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-19T20:28:51.524 INFO:teuthology.run_tasks:Running task ceph... 2026-02-19T20:28:51.535 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-19T20:28:51.536 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-19T20:28:51.536 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645', 'branch': 'wip-yuri-testing-2026-02-18-1820', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '59645', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial', 'no_nested_subset': False, 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'wip-yuri-testing-2026-02-18-1820'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri-testing-2026-02-18-1820', 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9494, 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri-testing-2026-02-18-1820', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_802bf111cd55e57f5842153d7ba2b8b964aa818a/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a', 'targets': {'trial013.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAjNtrYticDCX9pnZhRVVnflbZnQIugXBMd8zIps+2p71TYf4S3fwnv57bFRq5XnJ9JYrboYaeEBqKxKhQY5nNI=', 'trial151.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIjmP8lkx+u8iIeShbMgEV9ajpbAMXWwo0tojC0hZTyrhgo4Ro+8UWsRhCu8C++sBWAZLI6gwDwoff55PAEivAM='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '802bf111cd55e57f5842153d7ba2b8b964aa818a'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-19_15:47:56', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699'} 2026-02-19T20:28:51.536 DEBUG:teuthology.orchestra.run.trial013:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-19T20:28:51.537 DEBUG:teuthology.orchestra.run.trial151:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-19T20:28:51.541 DEBUG:teuthology.orchestra.run.trial013:> sudo install -d -m0777 -- /var/run/ceph 2026-02-19T20:28:51.583 DEBUG:teuthology.orchestra.run.trial151:> sudo install -d -m0777 -- /var/run/ceph 2026-02-19T20:28:51.593 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:51.593 DEBUG:teuthology.orchestra.run.trial151:> dd if=/scratch_devs of=/dev/stdout 2026-02-19T20:28:51.640 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-19T20:28:51.640 DEBUG:teuthology.orchestra.run.trial151:> stat /dev/vg_nvme/lv_1 2026-02-19T20:28:51.684 INFO:teuthology.orchestra.run.trial151.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-19T20:28:51.684 INFO:teuthology.orchestra.run.trial151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-19T20:28:51.684 INFO:teuthology.orchestra.run.trial151.stdout:Device: 5h/5d Inode: 903 Links: 1 2026-02-19T20:28:51.684 INFO:teuthology.orchestra.run.trial151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-19T20:28:51.684 INFO:teuthology.orchestra.run.trial151.stdout:Access: 2026-02-19 20:27:12.431587186 +0000 2026-02-19T20:28:51.684 INFO:teuthology.orchestra.run.trial151.stdout:Modify: 2026-02-19 20:27:12.275589839 +0000 2026-02-19T20:28:51.684 INFO:teuthology.orchestra.run.trial151.stdout:Change: 2026-02-19 20:27:12.275589839 +0000 2026-02-19T20:28:51.684 INFO:teuthology.orchestra.run.trial151.stdout: Birth: - 2026-02-19T20:28:51.684 DEBUG:teuthology.orchestra.run.trial151:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-19T20:28:51.731 INFO:teuthology.orchestra.run.trial151.stderr:1+0 records in 2026-02-19T20:28:51.731 INFO:teuthology.orchestra.run.trial151.stderr:1+0 records out 2026-02-19T20:28:51.731 INFO:teuthology.orchestra.run.trial151.stderr:512 bytes copied, 0.000174705 s, 2.9 MB/s 2026-02-19T20:28:51.732 DEBUG:teuthology.orchestra.run.trial151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-19T20:28:51.778 DEBUG:teuthology.orchestra.run.trial151:> stat /dev/vg_nvme/lv_2 2026-02-19T20:28:51.825 INFO:teuthology.orchestra.run.trial151.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-19T20:28:51.825 INFO:teuthology.orchestra.run.trial151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-19T20:28:51.825 INFO:teuthology.orchestra.run.trial151.stdout:Device: 5h/5d Inode: 929 Links: 1 2026-02-19T20:28:51.825 INFO:teuthology.orchestra.run.trial151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-19T20:28:51.825 INFO:teuthology.orchestra.run.trial151.stdout:Access: 2026-02-19 20:27:12.799580928 +0000 2026-02-19T20:28:51.825 INFO:teuthology.orchestra.run.trial151.stdout:Modify: 2026-02-19 20:27:12.619583990 +0000 2026-02-19T20:28:51.825 INFO:teuthology.orchestra.run.trial151.stdout:Change: 2026-02-19 20:27:12.619583990 +0000 2026-02-19T20:28:51.825 INFO:teuthology.orchestra.run.trial151.stdout: Birth: - 2026-02-19T20:28:51.826 DEBUG:teuthology.orchestra.run.trial151:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-19T20:28:51.876 INFO:teuthology.orchestra.run.trial151.stderr:1+0 records in 2026-02-19T20:28:51.876 INFO:teuthology.orchestra.run.trial151.stderr:1+0 records out 2026-02-19T20:28:51.876 INFO:teuthology.orchestra.run.trial151.stderr:512 bytes copied, 0.000165468 s, 3.1 MB/s 2026-02-19T20:28:51.877 DEBUG:teuthology.orchestra.run.trial151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-19T20:28:51.926 DEBUG:teuthology.orchestra.run.trial151:> stat /dev/vg_nvme/lv_3 2026-02-19T20:28:51.972 INFO:teuthology.orchestra.run.trial151.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-19T20:28:51.973 INFO:teuthology.orchestra.run.trial151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-19T20:28:51.973 INFO:teuthology.orchestra.run.trial151.stdout:Device: 5h/5d Inode: 951 Links: 1 2026-02-19T20:28:51.973 INFO:teuthology.orchestra.run.trial151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-19T20:28:51.973 INFO:teuthology.orchestra.run.trial151.stdout:Access: 2026-02-19 20:27:12.959578208 +0000 2026-02-19T20:28:51.973 INFO:teuthology.orchestra.run.trial151.stdout:Modify: 2026-02-19 20:27:12.955578278 +0000 2026-02-19T20:28:51.973 INFO:teuthology.orchestra.run.trial151.stdout:Change: 2026-02-19 20:27:12.955578278 +0000 2026-02-19T20:28:51.973 INFO:teuthology.orchestra.run.trial151.stdout: Birth: - 2026-02-19T20:28:51.973 DEBUG:teuthology.orchestra.run.trial151:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-19T20:28:52.020 INFO:teuthology.orchestra.run.trial151.stderr:1+0 records in 2026-02-19T20:28:52.020 INFO:teuthology.orchestra.run.trial151.stderr:1+0 records out 2026-02-19T20:28:52.020 INFO:teuthology.orchestra.run.trial151.stderr:512 bytes copied, 0.000186947 s, 2.7 MB/s 2026-02-19T20:28:52.021 DEBUG:teuthology.orchestra.run.trial151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-19T20:28:52.070 DEBUG:teuthology.orchestra.run.trial151:> stat /dev/vg_nvme/lv_4 2026-02-19T20:28:52.121 INFO:teuthology.orchestra.run.trial151.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-19T20:28:52.121 INFO:teuthology.orchestra.run.trial151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-19T20:28:52.121 INFO:teuthology.orchestra.run.trial151.stdout:Device: 5h/5d Inode: 975 Links: 1 2026-02-19T20:28:52.121 INFO:teuthology.orchestra.run.trial151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-19T20:28:52.121 INFO:teuthology.orchestra.run.trial151.stdout:Access: 2026-02-19 20:27:13.519568689 +0000 2026-02-19T20:28:52.121 INFO:teuthology.orchestra.run.trial151.stdout:Modify: 2026-02-19 20:27:13.331571885 +0000 2026-02-19T20:28:52.121 INFO:teuthology.orchestra.run.trial151.stdout:Change: 2026-02-19 20:27:13.331571885 +0000 2026-02-19T20:28:52.121 INFO:teuthology.orchestra.run.trial151.stdout: Birth: - 2026-02-19T20:28:52.122 DEBUG:teuthology.orchestra.run.trial151:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-19T20:28:52.172 INFO:teuthology.orchestra.run.trial151.stderr:1+0 records in 2026-02-19T20:28:52.172 INFO:teuthology.orchestra.run.trial151.stderr:1+0 records out 2026-02-19T20:28:52.172 INFO:teuthology.orchestra.run.trial151.stderr:512 bytes copied, 0.000163313 s, 3.1 MB/s 2026-02-19T20:28:52.173 DEBUG:teuthology.orchestra.run.trial151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-19T20:28:52.222 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-19T20:28:52.222 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial151.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-19T20:28:52.222 INFO:tasks.ceph:Generating config... 2026-02-19T20:28:52.225 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-19T20:28:52.225 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-19T20:28:52.225 INFO:tasks.ceph:[client] log to stderr = False 2026-02-19T20:28:52.225 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-19T20:28:52.225 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-02-19T20:28:52.225 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.225 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.225 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.226 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.226 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.226 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.226 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.226 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.226 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-19T20:28:52.226 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-19T20:28:52.226 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-19T20:28:52.226 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-19T20:28:52.226 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-19T20:28:52.226 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-19T20:28:52.227 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-19T20:28:52.227 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-19T20:28:52.227 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2026-02-19T20:28:52.227 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2026-02-19T20:28:52.227 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-19T20:28:52.227 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-19T20:28:52.227 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-19T20:28:52.227 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-19T20:28:52.227 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-19T20:28:52.227 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-19T20:28:52.227 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-19T20:28:52.227 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-19T20:28:52.228 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-19T20:28:52.228 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-19T20:28:52.228 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-19T20:28:52.228 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-19T20:28:52.228 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-19T20:28:52.228 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-19T20:28:52.281 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster2.keyring 2026-02-19T20:28:52.283 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-19T20:28:52.348 DEBUG:teuthology.orchestra.run.trial151:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-19T20:28:52.396 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.151')] 2026-02-19T20:28:52.397 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.151', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-19T20:28:52.398 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:52.398 DEBUG:teuthology.orchestra.run.trial151:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-19T20:28:52.441 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.151 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-19T20:28:52.497 INFO:teuthology.orchestra.run.trial151.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-19T20:28:52.498 INFO:teuthology.orchestra.run.trial151.stdout:monmaptool: generated fsid a14e4c04-90a3-4ebc-956c-42208d4170cb 2026-02-19T20:28:52.498 INFO:teuthology.orchestra.run.trial151.stdout:setting min_mon_release = tentacle 2026-02-19T20:28:52.498 INFO:teuthology.orchestra.run.trial151.stdout:epoch 0 2026-02-19T20:28:52.498 INFO:teuthology.orchestra.run.trial151.stdout:fsid a14e4c04-90a3-4ebc-956c-42208d4170cb 2026-02-19T20:28:52.498 INFO:teuthology.orchestra.run.trial151.stdout:last_changed 2026-02-19T20:28:52.495466+0000 2026-02-19T20:28:52.498 INFO:teuthology.orchestra.run.trial151.stdout:created 2026-02-19T20:28:52.495466+0000 2026-02-19T20:28:52.498 INFO:teuthology.orchestra.run.trial151.stdout:min_mon_release 20 (tentacle) 2026-02-19T20:28:52.498 INFO:teuthology.orchestra.run.trial151.stdout:election_strategy: 1 2026-02-19T20:28:52.498 INFO:teuthology.orchestra.run.trial151.stdout:0: [v2:10.20.193.151:3300/0,v1:10.20.193.151:6789/0] mon.a 2026-02-19T20:28:52.499 INFO:teuthology.orchestra.run.trial151.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-19T20:28:52.500 DEBUG:teuthology.orchestra.run.trial151:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-19T20:28:52.544 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID a14e4c04-90a3-4ebc-956c-42208d4170cb... 2026-02-19T20:28:52.546 DEBUG:teuthology.orchestra.run.trial013:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-19T20:28:52.548 DEBUG:teuthology.orchestra.run.trial151:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-19T20:28:52.587 INFO:teuthology.orchestra.run.trial013.stdout:[global] 2026-02-19T20:28:52.587 INFO:teuthology.orchestra.run.trial013.stdout: chdir = "" 2026-02-19T20:28:52.587 INFO:teuthology.orchestra.run.trial013.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-19T20:28:52.587 INFO:teuthology.orchestra.run.trial013.stdout: auth supported = cephx 2026-02-19T20:28:52.587 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: filestore xattr use omap = true 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: mon clock drift allowed = 1.000 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: osd crush chooseleaf type = 0 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: auth debug = true 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: ms die on old message = true 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: ms die on bug = true 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.588 INFO:teuthology.orchestra.run.trial013.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: mon pg warn max object skew = 0 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: # disable pg_autoscaler by default for new pools 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: osd pool default size = 2 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: mon osd allow primary affinity = true 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: mon osd allow pg remap = true 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on legacy crush tunables = false 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on crush straw calc version zero = false 2026-02-19T20:28:52.589 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on no sortbitwise = false 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on osd down out interval zero = false 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on too few osds = false 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: mon_allow_pool_size_one = true 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: osd default data pool replay window = 5 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: mon allow pool delete = true 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.590 INFO:teuthology.orchestra.run.trial013.stdout: mon cluster log file level = debug 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: debug asserts on shutdown = true 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: mon health detail to clog = false 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: mon host = 10.20.193.151 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: mon client directed command retry = 5 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: mon warn on pool no app = False 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: ms inject socket failures = 5000 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: fsid = a14e4c04-90a3-4ebc-956c-42208d4170cb 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout:[osd] 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: osd journal size = 100 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.591 INFO:teuthology.orchestra.run.trial013.stdout: osd scrub load threshold = 5.0 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd scrub max interval = 600 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd mclock profile = high_recovery_ops 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd mclock skip benchmark = true 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd recover clone overlap = true 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd recovery max chunk = 1048576 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd debug shutdown = true 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd debug op order = true 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd debug verify stray on activate = true 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd debug trim objects = true 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd open classes on start = true 2026-02-19T20:28:52.592 INFO:teuthology.orchestra.run.trial013.stdout: osd debug pg log writeout = true 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: osd deep scrub update digest min age = 30 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: osd map max advance = 10 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: journal zero on create = true 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: filestore ondisk finisher threads = 3 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: filestore apply finisher threads = 3 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: bdev debug aio = true 2026-02-19T20:28:52.593 INFO:teuthology.orchestra.run.trial013.stdout: osd debug misdirected ops = true 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: bluefs allocator = hybrid 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: bluestore allocator = hybrid 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: bluestore block size = 96636764160 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: bluestore fsck on mount = True 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: debug bluefs = 1/20 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: debug bluestore = 1/20 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: debug ms = 1 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: debug osd = 20 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: debug rocksdb = 4/10 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: mon osd backfillfull_ratio = 0.85 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: mon osd full ratio = 0.9 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: mon osd nearfull ratio = 0.8 2026-02-19T20:28:52.594 INFO:teuthology.orchestra.run.trial013.stdout: osd failsafe full ratio = 0.95 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: osd objectstore = bluestore 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout:[mgr] 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: debug ms = 1 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: debug mgr = 20 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: debug mon = 20 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: debug auth = 20 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: mon reweight min pgs per osd = 4 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: mon reweight min bytes per osd = 10 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: mgr/telemetry/nag = false 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout:[mon] 2026-02-19T20:28:52.595 INFO:teuthology.orchestra.run.trial013.stdout: debug ms = 1 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: debug mon = 20 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: debug paxos = 20 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: debug auth = 20 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: mon data avail warn = 5 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: mon mgr mkfs grace = 240 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: mon reweight min pgs per osd = 4 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: mon osd reporter subtree level = osd 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: mon osd prime pg temp = true 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: mon reweight min bytes per osd = 10 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: auth mon ticket ttl = 660 # 11m 2026-02-19T20:28:52.596 INFO:teuthology.orchestra.run.trial013.stdout: auth service ticket ttl = 240 # 4m 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: # don't complain about insecure global_id in the test suite 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: # 1m isn't quite enough 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: mon_down_mkfs_grace = 2m 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: mon_warn_on_filestore_osds = false 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout:[client] 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: rgw cache enabled = true 2026-02-19T20:28:52.597 INFO:teuthology.orchestra.run.trial013.stdout: rgw enable ops log = true 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout: rgw enable usage log = true 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout: debug rbd = 20 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout: debug rbd_mirror = 15 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout: log to stderr = False 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout: rbd default features = 125 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout: rbd mirror image policy type = simple 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout:[mon.a] 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.0] 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.598 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.1] 2026-02-19T20:28:52.599 INFO:teuthology.orchestra.run.trial013.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.599 INFO:teuthology.orchestra.run.trial013.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.599 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.2] 2026-02-19T20:28:52.599 INFO:teuthology.orchestra.run.trial013.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.599 INFO:teuthology.orchestra.run.trial013.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.599 INFO:teuthology.orchestra.run.trial013.stdout:[client.mirror.3] 2026-02-19T20:28:52.599 INFO:teuthology.orchestra.run.trial013.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.599 INFO:teuthology.orchestra.run.trial013.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.604 INFO:teuthology.orchestra.run.trial151.stdout:[global] 2026-02-19T20:28:52.604 INFO:teuthology.orchestra.run.trial151.stdout: chdir = "" 2026-02-19T20:28:52.604 INFO:teuthology.orchestra.run.trial151.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-19T20:28:52.604 INFO:teuthology.orchestra.run.trial151.stdout: auth supported = cephx 2026-02-19T20:28:52.604 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.604 INFO:teuthology.orchestra.run.trial151.stdout: filestore xattr use omap = true 2026-02-19T20:28:52.604 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: mon clock drift allowed = 1.000 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: osd crush chooseleaf type = 0 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: auth debug = true 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: ms die on old message = true 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: ms die on bug = true 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: mon pg warn max object skew = 0 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.605 INFO:teuthology.orchestra.run.trial151.stdout: # disable pg_autoscaler by default for new pools 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: osd pool default size = 2 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: mon osd allow primary affinity = true 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: mon osd allow pg remap = true 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on legacy crush tunables = false 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on crush straw calc version zero = false 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on no sortbitwise = false 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on osd down out interval zero = false 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on too few osds = false 2026-02-19T20:28:52.606 INFO:teuthology.orchestra.run.trial151.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: mon_allow_pool_size_one = true 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: osd default data pool replay window = 5 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: mon allow pool delete = true 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: mon cluster log file level = debug 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: debug asserts on shutdown = true 2026-02-19T20:28:52.607 INFO:teuthology.orchestra.run.trial151.stdout: mon health detail to clog = false 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: mon host = 10.20.193.151 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: mon client directed command retry = 5 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: mon warn on pool no app = False 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: ms inject socket failures = 5000 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: fsid = a14e4c04-90a3-4ebc-956c-42208d4170cb 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout:[osd] 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: osd journal size = 100 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: osd scrub load threshold = 5.0 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: osd scrub max interval = 600 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: osd mclock profile = high_recovery_ops 2026-02-19T20:28:52.608 INFO:teuthology.orchestra.run.trial151.stdout: osd mclock skip benchmark = true 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: osd recover clone overlap = true 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: osd recovery max chunk = 1048576 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: osd debug shutdown = true 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: osd debug op order = true 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: osd debug verify stray on activate = true 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: osd debug trim objects = true 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: osd open classes on start = true 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: osd debug pg log writeout = true 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.609 INFO:teuthology.orchestra.run.trial151.stdout: osd deep scrub update digest min age = 30 2026-02-19T20:28:52.610 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.610 INFO:teuthology.orchestra.run.trial151.stdout: osd map max advance = 10 2026-02-19T20:28:52.610 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.610 INFO:teuthology.orchestra.run.trial151.stdout: journal zero on create = true 2026-02-19T20:28:52.610 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.610 INFO:teuthology.orchestra.run.trial151.stdout: filestore ondisk finisher threads = 3 2026-02-19T20:28:52.610 INFO:teuthology.orchestra.run.trial151.stdout: filestore apply finisher threads = 3 2026-02-19T20:28:52.611 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.611 INFO:teuthology.orchestra.run.trial151.stdout: bdev debug aio = true 2026-02-19T20:28:52.611 INFO:teuthology.orchestra.run.trial151.stdout: osd debug misdirected ops = true 2026-02-19T20:28:52.611 INFO:teuthology.orchestra.run.trial151.stdout: bluefs allocator = hybrid 2026-02-19T20:28:52.611 INFO:teuthology.orchestra.run.trial151.stdout: bluestore allocator = hybrid 2026-02-19T20:28:52.611 INFO:teuthology.orchestra.run.trial151.stdout: bluestore block size = 96636764160 2026-02-19T20:28:52.612 INFO:teuthology.orchestra.run.trial151.stdout: bluestore fsck on mount = True 2026-02-19T20:28:52.612 INFO:teuthology.orchestra.run.trial151.stdout: debug bluefs = 1/20 2026-02-19T20:28:52.612 INFO:teuthology.orchestra.run.trial151.stdout: debug bluestore = 1/20 2026-02-19T20:28:52.612 INFO:teuthology.orchestra.run.trial151.stdout: debug ms = 1 2026-02-19T20:28:52.612 INFO:teuthology.orchestra.run.trial151.stdout: debug osd = 20 2026-02-19T20:28:52.612 INFO:teuthology.orchestra.run.trial151.stdout: debug rocksdb = 4/10 2026-02-19T20:28:52.613 INFO:teuthology.orchestra.run.trial151.stdout: mon osd backfillfull_ratio = 0.85 2026-02-19T20:28:52.613 INFO:teuthology.orchestra.run.trial151.stdout: mon osd full ratio = 0.9 2026-02-19T20:28:52.613 INFO:teuthology.orchestra.run.trial151.stdout: mon osd nearfull ratio = 0.8 2026-02-19T20:28:52.613 INFO:teuthology.orchestra.run.trial151.stdout: osd failsafe full ratio = 0.95 2026-02-19T20:28:52.613 INFO:teuthology.orchestra.run.trial151.stdout: osd objectstore = bluestore 2026-02-19T20:28:52.613 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.613 INFO:teuthology.orchestra.run.trial151.stdout:[mgr] 2026-02-19T20:28:52.613 INFO:teuthology.orchestra.run.trial151.stdout: debug ms = 1 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: debug mgr = 20 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: debug mon = 20 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: debug auth = 20 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: mon reweight min pgs per osd = 4 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: mon reweight min bytes per osd = 10 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: mgr/telemetry/nag = false 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout:[mon] 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: debug ms = 1 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: debug mon = 20 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: debug paxos = 20 2026-02-19T20:28:52.614 INFO:teuthology.orchestra.run.trial151.stdout: debug auth = 20 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: mon data avail warn = 5 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: mon mgr mkfs grace = 240 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: mon reweight min pgs per osd = 4 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: mon osd reporter subtree level = osd 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: mon osd prime pg temp = true 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: mon reweight min bytes per osd = 10 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: auth mon ticket ttl = 660 # 11m 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: auth service ticket ttl = 240 # 4m 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: # don't complain about insecure global_id in the test suite 2026-02-19T20:28:52.615 INFO:teuthology.orchestra.run.trial151.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: # 1m isn't quite enough 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: mon_down_mkfs_grace = 2m 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: mon_warn_on_filestore_osds = false 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout:[client] 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: rgw cache enabled = true 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: rgw enable ops log = true 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: rgw enable usage log = true 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-19T20:28:52.616 INFO:teuthology.orchestra.run.trial151.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout: debug rbd = 20 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout: debug rbd_mirror = 15 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout: log to stderr = False 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout: rbd default features = 125 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout: rbd mirror image policy type = simple 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout:[mon.a] 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.0] 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.1] 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.617 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.2] 2026-02-19T20:28:52.618 INFO:teuthology.orchestra.run.trial151.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.618 INFO:teuthology.orchestra.run.trial151.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.618 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.3] 2026-02-19T20:28:52.618 INFO:teuthology.orchestra.run.trial151.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-19T20:28:52.618 INFO:teuthology.orchestra.run.trial151.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-19T20:28:52.619 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-19T20:28:52.619 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-19T20:28:52.683 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-19T20:28:52.684 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:52.684 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-19T20:28:52.729 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:52.729 DEBUG:teuthology.orchestra.run.trial151:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-19T20:28:52.777 INFO:tasks.ceph:Sending monmap to node ubuntu@trial013.front.sepia.ceph.com 2026-02-19T20:28:52.777 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:52.777 DEBUG:teuthology.orchestra.run.trial013:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-19T20:28:52.777 DEBUG:teuthology.orchestra.run.trial013:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-19T20:28:52.792 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:52.792 DEBUG:teuthology.orchestra.run.trial013:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-19T20:28:52.838 INFO:tasks.ceph:Sending monmap to node ubuntu@trial151.front.sepia.ceph.com 2026-02-19T20:28:52.838 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:52.838 DEBUG:teuthology.orchestra.run.trial151:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-19T20:28:52.839 DEBUG:teuthology.orchestra.run.trial151:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-19T20:28:52.853 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:52.853 DEBUG:teuthology.orchestra.run.trial151:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-19T20:28:52.902 INFO:tasks.ceph:Setting up mon nodes... 2026-02-19T20:28:52.902 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-19T20:28:52.902 DEBUG:teuthology.orchestra.run.trial013:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-19T20:28:52.932 INFO:teuthology.orchestra.run.trial013.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-19T20:28:52.935 INFO:tasks.ceph:Setting up mds nodes... 2026-02-19T20:28:52.935 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-19T20:28:52.935 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-19T20:28:52.995 INFO:teuthology.orchestra.run.trial013.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-19T20:28:53.002 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-19T20:28:53.022 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-19T20:28:53.029 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-19T20:28:53.095 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-19T20:28:53.103 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-19T20:28:53.171 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-19T20:28:53.179 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-19T20:28:53.237 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-19T20:28:53.244 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-19T20:28:53.311 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-19T20:28:53.321 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-19T20:28:53.377 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-19T20:28:53.384 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-19T20:28:53.451 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-19T20:28:53.459 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-19T20:28:53.516 INFO:teuthology.orchestra.run.trial151.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-19T20:28:53.523 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-19T20:28:53.524 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial013.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial151.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-19T20:28:53.524 DEBUG:teuthology.orchestra.run.trial151:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-19T20:28:53.576 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-19T20:28:53.577 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-19T20:28:53.577 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial151.front.sepia.ceph.com 2026-02-19T20:28:53.577 DEBUG:teuthology.orchestra.run.trial151:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-19T20:28:53.626 INFO:teuthology.orchestra.run.trial151.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-19T20:28:53.626 INFO:teuthology.orchestra.run.trial151.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-19T20:28:53.626 INFO:teuthology.orchestra.run.trial151.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-19T20:28:53.627 INFO:teuthology.orchestra.run.trial151.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-19T20:28:53.627 INFO:teuthology.orchestra.run.trial151.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-19T20:28:53.627 INFO:teuthology.orchestra.run.trial151.stdout: = sunit=0 swidth=0 blks 2026-02-19T20:28:53.627 INFO:teuthology.orchestra.run.trial151.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-19T20:28:53.627 INFO:teuthology.orchestra.run.trial151.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-19T20:28:53.627 INFO:teuthology.orchestra.run.trial151.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-19T20:28:53.627 INFO:teuthology.orchestra.run.trial151.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-19T20:28:54.248 INFO:teuthology.orchestra.run.trial151.stdout:Discarding blocks...Done. 2026-02-19T20:28:54.250 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial151.front.sepia.ceph.com -o noatime 2026-02-19T20:28:54.250 DEBUG:teuthology.orchestra.run.trial151:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-19T20:28:54.309 DEBUG:teuthology.orchestra.run.trial151:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-19T20:28:54.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T20:28:54.359 INFO:teuthology.orchestra.run.trial151.stderr:sudo: /sbin/restorecon: command not found 2026-02-19T20:28:54.360 DEBUG:teuthology.orchestra.run.trial151:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-19T20:28:54.409 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-19T20:28:54.409 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-19T20:28:54.409 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial151.front.sepia.ceph.com 2026-02-19T20:28:54.410 DEBUG:teuthology.orchestra.run.trial151:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-19T20:28:54.463 INFO:teuthology.orchestra.run.trial151.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-19T20:28:54.463 INFO:teuthology.orchestra.run.trial151.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-19T20:28:54.463 INFO:teuthology.orchestra.run.trial151.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-19T20:28:54.463 INFO:teuthology.orchestra.run.trial151.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-19T20:28:54.463 INFO:teuthology.orchestra.run.trial151.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-19T20:28:54.463 INFO:teuthology.orchestra.run.trial151.stdout: = sunit=0 swidth=0 blks 2026-02-19T20:28:54.464 INFO:teuthology.orchestra.run.trial151.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-19T20:28:54.464 INFO:teuthology.orchestra.run.trial151.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-19T20:28:54.464 INFO:teuthology.orchestra.run.trial151.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-19T20:28:54.464 INFO:teuthology.orchestra.run.trial151.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-19T20:28:55.025 INFO:teuthology.orchestra.run.trial151.stdout:Discarding blocks...Done. 2026-02-19T20:28:55.027 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial151.front.sepia.ceph.com -o noatime 2026-02-19T20:28:55.027 DEBUG:teuthology.orchestra.run.trial151:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-19T20:28:55.085 DEBUG:teuthology.orchestra.run.trial151:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-19T20:28:55.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T20:28:55.135 INFO:teuthology.orchestra.run.trial151.stderr:sudo: /sbin/restorecon: command not found 2026-02-19T20:28:55.135 DEBUG:teuthology.orchestra.run.trial151:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-19T20:28:55.185 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-19T20:28:55.185 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-19T20:28:55.185 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial151.front.sepia.ceph.com 2026-02-19T20:28:55.185 DEBUG:teuthology.orchestra.run.trial151:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-19T20:28:55.234 INFO:teuthology.orchestra.run.trial151.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-19T20:28:55.234 INFO:teuthology.orchestra.run.trial151.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-19T20:28:55.235 INFO:teuthology.orchestra.run.trial151.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-19T20:28:55.235 INFO:teuthology.orchestra.run.trial151.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-19T20:28:55.235 INFO:teuthology.orchestra.run.trial151.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-19T20:28:55.235 INFO:teuthology.orchestra.run.trial151.stdout: = sunit=0 swidth=0 blks 2026-02-19T20:28:55.235 INFO:teuthology.orchestra.run.trial151.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-19T20:28:55.235 INFO:teuthology.orchestra.run.trial151.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-19T20:28:55.235 INFO:teuthology.orchestra.run.trial151.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-19T20:28:55.235 INFO:teuthology.orchestra.run.trial151.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-19T20:28:55.847 INFO:teuthology.orchestra.run.trial151.stdout:Discarding blocks...Done. 2026-02-19T20:28:55.848 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial151.front.sepia.ceph.com -o noatime 2026-02-19T20:28:55.849 DEBUG:teuthology.orchestra.run.trial151:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-19T20:28:55.906 DEBUG:teuthology.orchestra.run.trial151:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-19T20:28:55.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T20:28:55.955 INFO:teuthology.orchestra.run.trial151.stderr:sudo: /sbin/restorecon: command not found 2026-02-19T20:28:55.956 DEBUG:teuthology.orchestra.run.trial151:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-19T20:28:56.018 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:56.014+0000 7f0c7db89a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-19T20:28:56.019 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:56.014+0000 7f0c7db89a40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-19T20:28:56.019 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:56.014+0000 7f0c7db89a40 -1 bdev(0x55d8ce25d800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-19T20:28:56.019 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:56.014+0000 7f0c7db89a40 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-19T20:28:56.862 DEBUG:teuthology.orchestra.run.trial151:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-19T20:28:56.917 DEBUG:teuthology.orchestra.run.trial151:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-19T20:28:56.982 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:56.978+0000 7f9c70ed8a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-19T20:28:56.983 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:56.978+0000 7f9c70ed8a40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-19T20:28:56.983 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:56.978+0000 7f9c70ed8a40 -1 bdev(0x55f29c299800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-19T20:28:56.983 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:56.978+0000 7f9c70ed8a40 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-19T20:28:57.903 DEBUG:teuthology.orchestra.run.trial151:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-19T20:28:57.954 DEBUG:teuthology.orchestra.run.trial151:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-19T20:28:58.018 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:58.014+0000 7f3a6381aa40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-19T20:28:58.019 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:58.014+0000 7f3a6381aa40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-19T20:28:58.019 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:58.014+0000 7f3a6381aa40 -1 bdev(0x55f2672a9800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-19T20:28:58.019 INFO:teuthology.orchestra.run.trial151.stderr:2026-02-19T20:28:58.014+0000 7f3a6381aa40 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-19T20:28:58.901 DEBUG:teuthology.orchestra.run.trial151:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-19T20:28:58.958 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-19T20:28:58.958 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:58.958 DEBUG:teuthology.orchestra.run.trial013:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-19T20:28:58.969 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:58.969 DEBUG:teuthology.orchestra.run.trial151:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-19T20:28:59.010 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:59.010 DEBUG:teuthology.orchestra.run.trial151:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-19T20:28:59.062 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:59.062 DEBUG:teuthology.orchestra.run.trial151:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-19T20:28:59.112 DEBUG:teuthology.orchestra.run.trial013:> set -ex 2026-02-19T20:28:59.113 DEBUG:teuthology.orchestra.run.trial013:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-19T20:28:59.119 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:59.119 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-19T20:28:59.157 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:59.157 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-19T20:28:59.206 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:59.206 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-19T20:28:59.254 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:59.254 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-19T20:28:59.301 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:59.301 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-19T20:28:59.349 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:59.349 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-19T20:28:59.393 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:59.393 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-19T20:28:59.437 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:28:59.437 DEBUG:teuthology.orchestra.run.trial151:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-19T20:28:59.485 INFO:tasks.ceph:Adding keys to all mons... 2026-02-19T20:28:59.485 DEBUG:teuthology.orchestra.run.trial151:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-19T20:28:59.536 INFO:teuthology.orchestra.run.trial151.stdout:[mgr.x] 2026-02-19T20:28:59.536 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCEcpdp/QqqNxAAkRLgRT30V7roy8bwVQdz4A== 2026-02-19T20:28:59.536 INFO:teuthology.orchestra.run.trial151.stdout:[osd.0] 2026-02-19T20:28:59.536 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCIcpdprUX6ABAAFa19PL/Pb8AFKE/AmRLExw== 2026-02-19T20:28:59.536 INFO:teuthology.orchestra.run.trial151.stdout:[osd.1] 2026-02-19T20:28:59.536 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCIcpdpyIlmOhAA5FVZLByXDK1gVXX0rxzxcg== 2026-02-19T20:28:59.536 INFO:teuthology.orchestra.run.trial151.stdout:[osd.2] 2026-02-19T20:28:59.536 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCKcpdpw6j5ABAAzcZ9pie1KuQa0cx9XMtbng== 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout:[client.0] 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCEcpdpGlV1OxAAG44GDriosni+Zwswtjj5Zg== 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror] 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdp1PcxARAAn8bScqLX1zuDJCN57TxoIQ== 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.0] 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpUTKSBRAAQ0nzucVNeILjwRb+UzVA2g== 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.1] 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpI4UYChAAFYcC7W3z6tQR0h+ffSxRgQ== 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.2] 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpgAAHDhAA74fPazFQufybPeEnyUWyyg== 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.3] 2026-02-19T20:28:59.537 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpOcpuEhAALg8CtOV+umSUh/LreuYuwA== 2026-02-19T20:28:59.538 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.4] 2026-02-19T20:28:59.538 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpwdxgFhAANNQI7rjqNHJS9lidULtWRg== 2026-02-19T20:28:59.538 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.5] 2026-02-19T20:28:59.538 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpGN3JGhAAbLuKGqjwQrA0iOg1e/2vhA== 2026-02-19T20:28:59.538 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.6] 2026-02-19T20:28:59.538 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpWmavHhAAra24WLXguMdTMvmwPHq/hQ== 2026-02-19T20:28:59.539 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-19T20:28:59.603 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-19T20:28:59.666 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-19T20:28:59.730 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-19T20:28:59.795 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:59.858 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:59.923 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:28:59.988 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:29:00.051 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:29:00.117 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:29:00.180 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:29:00.244 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:29:00.308 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-19T20:29:00.371 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-19T20:29:00.371 DEBUG:teuthology.orchestra.run.trial151:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-19T20:29:00.420 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-19T20:29:00.491 DEBUG:teuthology.orchestra.run.trial151:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-19T20:29:00.541 DEBUG:teuthology.orchestra.run.trial151:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-19T20:29:00.592 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-19T20:29:00.593 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-19T20:29:00.593 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-19T20:29:00.634 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-19T20:29:00.635 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-19T20:29:00.635 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-19T20:29:00.635 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-19T20:29:00.637 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-19T20:29:00.638 DEBUG:tasks.ceph:set 0 configs 2026-02-19T20:29:00.638 DEBUG:teuthology.orchestra.run.trial151:> sudo ceph --cluster cluster2 config dump 2026-02-19T20:29:00.756 INFO:teuthology.orchestra.run.trial151.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-19T20:29:00.768 INFO:tasks.ceph:Setting crush tunables to default 2026-02-19T20:29:00.768 DEBUG:teuthology.orchestra.run.trial151:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-19T20:29:00.865 INFO:teuthology.orchestra.run.trial151.stderr:adjusted tunables profile to default 2026-02-19T20:29:00.878 INFO:tasks.ceph:check_enable_crimson: False 2026-02-19T20:29:00.878 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-19T20:29:00.879 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:29:00.879 DEBUG:teuthology.orchestra.run.trial151:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-19T20:29:00.889 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:29:00.889 DEBUG:teuthology.orchestra.run.trial151:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-19T20:29:00.941 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:29:00.941 DEBUG:teuthology.orchestra.run.trial151:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-19T20:29:00.994 DEBUG:teuthology.orchestra.run.trial151:> sudo ceph --cluster cluster2 osd new ddaf41b0-845c-4123-af03-f7dace7c6d41 0 2026-02-19T20:29:01.139 INFO:teuthology.orchestra.run.trial151.stdout:0 2026-02-19T20:29:01.149 DEBUG:teuthology.orchestra.run.trial151:> sudo ceph --cluster cluster2 osd new 31b161d3-699c-4d78-bf2f-2fe95de919b4 1 2026-02-19T20:29:01.244 INFO:teuthology.orchestra.run.trial151.stdout:1 2026-02-19T20:29:01.256 DEBUG:teuthology.orchestra.run.trial151:> sudo ceph --cluster cluster2 osd new e835eb22-9891-4406-8ec0-05ccd24a9828 2 2026-02-19T20:29:01.354 INFO:teuthology.orchestra.run.trial151.stdout:2 2026-02-19T20:29:01.363 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-19T20:29:01.364 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-19T20:29:01.365 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-19T20:29:01.366 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-19T20:29:01.366 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-19T20:29:01.368 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-19T20:29:01.368 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-19T20:29:01.368 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-19T20:29:01.370 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-19T20:29:01.370 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-19T20:29:01.465 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:29:01.465 INFO:teuthology.orchestra.run.trial151.stdout:{"epoch":5,"fsid":"a14e4c04-90a3-4ebc-956c-42208d4170cb","created":"2026-02-19T20:29:00.712510+0000","modified":"2026-02-19T20:29:01.351350+0000","last_up_change":"0.000000","last_in_change":"2026-02-19T20:29:01.351350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ddaf41b0-845c-4123-af03-f7dace7c6d41","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"31b161d3-699c-4d78-bf2f-2fe95de919b4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"e835eb22-9891-4406-8ec0-05ccd24a9828","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T20:29:01.474 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-19T20:29:01.474 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-19T20:29:01.777 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-19T20:29:01.814 INFO:tasks.ceph.cluster2.osd.1.trial151.stderr:2026-02-19T20:29:01.810+0000 7f58fd0c9a40 -1 Falling back to public interface 2026-02-19T20:29:01.834 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:29:01.830+0000 7f786d101a40 -1 Falling back to public interface 2026-02-19T20:29:01.850 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:29:01.846+0000 7f5cf45baa40 -1 Falling back to public interface 2026-02-19T20:29:01.867 INFO:teuthology.misc.health.trial151.stdout: 2026-02-19T20:29:01.867 INFO:teuthology.misc.health.trial151.stdout:{"epoch":5,"fsid":"a14e4c04-90a3-4ebc-956c-42208d4170cb","created":"2026-02-19T20:29:00.712510+0000","modified":"2026-02-19T20:29:01.351350+0000","last_up_change":"0.000000","last_in_change":"2026-02-19T20:29:01.351350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ddaf41b0-845c-4123-af03-f7dace7c6d41","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"31b161d3-699c-4d78-bf2f-2fe95de919b4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"e835eb22-9891-4406-8ec0-05ccd24a9828","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T20:29:01.876 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-19T20:29:01.968 INFO:tasks.ceph.cluster2.mgr.x.trial013.stderr:ERROR:root:Module 'jmespath' is not installed. 2026-02-19T20:29:02.007 INFO:tasks.ceph.cluster2.mgr.x.trial013.stderr:ERROR:dashboard.controllers.nvmeof:Failed to import NVMeoFClient and related components: cannot import name 'NVMeoFClient' from 'dashboard.services.nvmeof_client' (/usr/share/ceph/mgr/dashboard/services/nvmeof_client.py) 2026-02-19T20:29:02.008 INFO:tasks.ceph.cluster2.mgr.x.trial013.stderr:ERROR:sso:Module 'jmespath' is not installed. 2026-02-19T20:29:02.023 INFO:tasks.ceph.cluster2.mgr.x.trial013.stderr:ERROR:root:Module 'xmltodict' is not installed. 2026-02-19T20:29:02.378 INFO:tasks.ceph.cluster2.mgr.x.trial013.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-19T20:29:02.378 INFO:tasks.ceph.cluster2.mgr.x.trial013.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-19T20:29:02.378 INFO:tasks.ceph.cluster2.mgr.x.trial013.stderr: from numpy import show_config as show_numpy_config 2026-02-19T20:29:02.948 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:29:02.942+0000 7f786d101a40 -1 osd.0 0 log_to_monitors true 2026-02-19T20:29:02.961 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:29:02.958+0000 7f5cf45baa40 -1 osd.2 0 log_to_monitors true 2026-02-19T20:29:02.998 INFO:tasks.ceph.cluster2.osd.1.trial151.stderr:2026-02-19T20:29:02.994+0000 7f58fd0c9a40 -1 osd.1 0 log_to_monitors true 2026-02-19T20:29:03.456 INFO:tasks.ceph.cluster2.mgr.x.trial013.stderr:2026-02-19T20:29:03.452+0000 7fafe0191640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-19T20:29:04.729 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:29:04.726+0000 7f5cf0564640 -1 osd.2 0 waiting for initial osdmap 2026-02-19T20:29:04.730 INFO:tasks.ceph.cluster2.osd.1.trial151.stderr:2026-02-19T20:29:04.726+0000 7f58f9073640 -1 osd.1 0 waiting for initial osdmap 2026-02-19T20:29:04.731 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:29:04.726+0000 7f78690ab640 -1 osd.0 0 waiting for initial osdmap 2026-02-19T20:29:04.732 INFO:tasks.ceph.cluster2.osd.1.trial151.stderr:2026-02-19T20:29:04.726+0000 7f58f0742640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-19T20:29:04.732 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:29:04.726+0000 7f5ce7c33640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-19T20:29:04.733 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:29:04.726+0000 7f786077a640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-19T20:29:08.178 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-19T20:29:08.325 INFO:teuthology.misc.health.trial151.stdout: 2026-02-19T20:29:08.326 INFO:teuthology.misc.health.trial151.stdout:{"epoch":10,"fsid":"a14e4c04-90a3-4ebc-956c-42208d4170cb","created":"2026-02-19T20:29:00.712510+0000","modified":"2026-02-19T20:29:07.730500+0000","last_up_change":"2026-02-19T20:29:05.723698+0000","last_in_change":"2026-02-19T20:29:01.351350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-19T20:29:06.483311+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ddaf41b0-845c-4123-af03-f7dace7c6d41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6808","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6809","nonce":1469645752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6810","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6811","nonce":1469645752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6814","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6815","nonce":1469645752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6812","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6813","nonce":1469645752}]},"public_addr":"10.20.193.151:6809/1469645752","cluster_addr":"10.20.193.151:6811/1469645752","heartbeat_back_addr":"10.20.193.151:6815/1469645752","heartbeat_front_addr":"10.20.193.151:6813/1469645752","state":["exists","up"]},{"osd":1,"uuid":"31b161d3-699c-4d78-bf2f-2fe95de919b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6800","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6801","nonce":3847550616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6802","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6803","nonce":3847550616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6806","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6807","nonce":3847550616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6804","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6805","nonce":3847550616}]},"public_addr":"10.20.193.151:6801/3847550616","cluster_addr":"10.20.193.151:6803/3847550616","heartbeat_back_addr":"10.20.193.151:6807/3847550616","heartbeat_front_addr":"10.20.193.151:6805/3847550616","state":["exists","up"]},{"osd":2,"uuid":"e835eb22-9891-4406-8ec0-05ccd24a9828","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6816","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6817","nonce":3305594166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6818","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6819","nonce":3305594166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6822","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6823","nonce":3305594166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6820","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6821","nonce":3305594166}]},"public_addr":"10.20.193.151:6817/3305594166","cluster_addr":"10.20.193.151:6819/3305594166","heartbeat_back_addr":"10.20.193.151:6823/3305594166","heartbeat_front_addr":"10.20.193.151:6821/3305594166","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T20:29:08.337 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-19T20:29:08.337 INFO:tasks.ceph:Creating RBD pool 2026-02-19T20:29:08.337 DEBUG:teuthology.orchestra.run.trial151:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-19T20:29:08.735 INFO:teuthology.orchestra.run.trial151.stderr:pool 'rbd' created 2026-02-19T20:29:08.751 DEBUG:teuthology.orchestra.run.trial151:> rbd --cluster cluster2 pool init rbd 2026-02-19T20:29:11.752 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-19T20:29:11.752 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2026-02-19T20:29:11.752 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-19T20:29:11.912 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:29:11.923 INFO:teuthology.orchestra.run.trial151.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-19T20:29:11.923 INFO:tasks.ceph_manager:config epoch is 1 2026-02-19T20:29:11.923 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-19T20:29:11.924 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-19T20:29:11.924 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-19T20:29:12.105 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:29:12.114 INFO:teuthology.orchestra.run.trial151.stdout:{"epoch":5,"flags":0,"active_gid":4099,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6826","nonce":1377989905},{"type":"v1","addr":"10.20.193.13:6827","nonce":1377989905}]},"active_addr":"10.20.193.13:6827/1377989905","active_change":"2026-02-19T20:29:02.445216+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often the agent on each host will gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"The first TCP port the agent will try to bind to. Up to 1000 subsequent ports will be attempted if this port cannot be bound","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this option can facilitate debugging daemons that encounter runtime problems.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"How frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"Fraction of total system memory to divide among autotuned daemons. Converged systems hosting both Ceph and compute should set a lower value; systems with more than 10 GiB of memory per OSD may benefit from a slightly larger value so that the abundance of RAM can be fully utilized","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self-signed certificates generated and signed by the cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently Podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_alloy":{"name":"container_image_alloy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/alloy:latest","min":"","max":"","enum_allowed":[],"desc":"Alloy container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.6","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"Seconds for which to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"Seconds for which to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"Seconds for which to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"Location of dashboards to include in Grafana","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"How frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether the hardware monitoring daemon be deployed on every host?","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at the debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"Max number of daemons per service per host: guard against mistakes and runaways","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"Max number of OSDs that will be drained simultaneously when OSDs are removed","long_desc":"","tags":[],"see_also":[]},"max_parallel_osd_upgrades":{"name":"max_parallel_osd_upgrades","type":"int","level":"advanced","flags":0,"default_value":"16","min":"","max":"","enum_allowed":[],"desc":"Maximum number of OSD daemons upgraded in parallel.","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"Mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default IP address for RedFish API (OOB management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"Location of alerts to include in Prometheus Alertmanager","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry URL for login purposes. This is not the default registry.","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"Cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Customized SSH config file to distribute to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many sequential SSH connection liveness check failures before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often SSH connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"How frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use the cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest to ensure that all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"healthcheck_history_max_entries":{"name":"healthcheck_history_max_entries","type":"int","level":"advanced","flags":1,"default_value":"1000","min":"","max":"","enum_allowed":[],"desc":"Maximum number of health check history entries to keep","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.13:0","nonce":641462344}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.13:0","nonce":4142934356}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.13:0","nonce":498735634}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.13:0","nonce":619950251}]}]} 2026-02-19T20:29:12.122 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-19T20:29:12.122 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-19T20:29:12.123 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-19T20:29:12.283 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:29:12.283 INFO:teuthology.orchestra.run.trial151.stdout:{"epoch":14,"fsid":"a14e4c04-90a3-4ebc-956c-42208d4170cb","created":"2026-02-19T20:29:00.712510+0000","modified":"2026-02-19T20:29:11.739422+0000","last_up_change":"2026-02-19T20:29:05.723698+0000","last_in_change":"2026-02-19T20:29:01.351350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-19T20:29:06.483311+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-19T20:29:08.492120+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ddaf41b0-845c-4123-af03-f7dace7c6d41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6808","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6809","nonce":1469645752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6810","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6811","nonce":1469645752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6814","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6815","nonce":1469645752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6812","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6813","nonce":1469645752}]},"public_addr":"10.20.193.151:6809/1469645752","cluster_addr":"10.20.193.151:6811/1469645752","heartbeat_back_addr":"10.20.193.151:6815/1469645752","heartbeat_front_addr":"10.20.193.151:6813/1469645752","state":["exists","up"]},{"osd":1,"uuid":"31b161d3-699c-4d78-bf2f-2fe95de919b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6800","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6801","nonce":3847550616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6802","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6803","nonce":3847550616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6806","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6807","nonce":3847550616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6804","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6805","nonce":3847550616}]},"public_addr":"10.20.193.151:6801/3847550616","cluster_addr":"10.20.193.151:6803/3847550616","heartbeat_back_addr":"10.20.193.151:6807/3847550616","heartbeat_front_addr":"10.20.193.151:6805/3847550616","state":["exists","up"]},{"osd":2,"uuid":"e835eb22-9891-4406-8ec0-05ccd24a9828","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6816","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6817","nonce":3305594166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6818","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6819","nonce":3305594166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6822","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6823","nonce":3305594166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6820","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6821","nonce":3305594166}]},"public_addr":"10.20.193.151:6817/3305594166","cluster_addr":"10.20.193.151:6819/3305594166","heartbeat_back_addr":"10.20.193.151:6823/3305594166","heartbeat_front_addr":"10.20.193.151:6821/3305594166","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T20:29:12.294 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-19T20:29:12.294 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-19T20:29:12.458 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:29:12.461 INFO:teuthology.orchestra.run.trial151.stdout:{"epoch":14,"fsid":"a14e4c04-90a3-4ebc-956c-42208d4170cb","created":"2026-02-19T20:29:00.712510+0000","modified":"2026-02-19T20:29:11.739422+0000","last_up_change":"2026-02-19T20:29:05.723698+0000","last_in_change":"2026-02-19T20:29:01.351350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-19T20:29:06.483311+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-19T20:29:08.492120+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ddaf41b0-845c-4123-af03-f7dace7c6d41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6808","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6809","nonce":1469645752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6810","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6811","nonce":1469645752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6814","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6815","nonce":1469645752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6812","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6813","nonce":1469645752}]},"public_addr":"10.20.193.151:6809/1469645752","cluster_addr":"10.20.193.151:6811/1469645752","heartbeat_back_addr":"10.20.193.151:6815/1469645752","heartbeat_front_addr":"10.20.193.151:6813/1469645752","state":["exists","up"]},{"osd":1,"uuid":"31b161d3-699c-4d78-bf2f-2fe95de919b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6800","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6801","nonce":3847550616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6802","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6803","nonce":3847550616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6806","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6807","nonce":3847550616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6804","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6805","nonce":3847550616}]},"public_addr":"10.20.193.151:6801/3847550616","cluster_addr":"10.20.193.151:6803/3847550616","heartbeat_back_addr":"10.20.193.151:6807/3847550616","heartbeat_front_addr":"10.20.193.151:6805/3847550616","state":["exists","up"]},{"osd":2,"uuid":"e835eb22-9891-4406-8ec0-05ccd24a9828","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6816","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6817","nonce":3305594166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6818","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6819","nonce":3305594166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6822","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6823","nonce":3305594166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6820","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6821","nonce":3305594166}]},"public_addr":"10.20.193.151:6817/3305594166","cluster_addr":"10.20.193.151:6819/3305594166","heartbeat_back_addr":"10.20.193.151:6823/3305594166","heartbeat_front_addr":"10.20.193.151:6821/3305594166","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T20:29:12.470 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-19T20:29:12.471 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-19T20:29:12.471 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-19T20:29:12.556 INFO:teuthology.orchestra.run.trial151.stdout:34359738372 2026-02-19T20:29:12.557 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-19T20:29:12.558 INFO:teuthology.orchestra.run.trial151.stdout:34359738372 2026-02-19T20:29:12.559 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-19T20:29:12.560 INFO:teuthology.orchestra.run.trial151.stdout:34359738371 2026-02-19T20:29:12.560 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-19T20:29:12.728 INFO:teuthology.orchestra.run.trial151.stdout:34359738370 2026-02-19T20:29:12.740 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738370 for osd.1 2026-02-19T20:29:12.756 INFO:teuthology.orchestra.run.trial151.stdout:34359738370 2026-02-19T20:29:12.759 INFO:teuthology.orchestra.run.trial151.stdout:34359738371 2026-02-19T20:29:12.767 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.0 2026-02-19T20:29:12.771 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.2 2026-02-19T20:29:13.741 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-19T20:29:13.768 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-19T20:29:13.772 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-19T20:29:13.906 INFO:teuthology.orchestra.run.trial151.stdout:34359738371 2026-02-19T20:29:13.918 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.1 2026-02-19T20:29:13.941 INFO:teuthology.orchestra.run.trial151.stdout:34359738370 2026-02-19T20:29:13.944 INFO:teuthology.orchestra.run.trial151.stdout:34359738371 2026-02-19T20:29:13.953 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.0 2026-02-19T20:29:13.956 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.2 2026-02-19T20:29:14.919 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-19T20:29:14.953 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-19T20:29:14.957 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-19T20:29:15.080 INFO:teuthology.orchestra.run.trial151.stdout:34359738372 2026-02-19T20:29:15.093 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.1 2026-02-19T20:29:15.093 DEBUG:teuthology.parallel:result is None 2026-02-19T20:29:15.113 INFO:teuthology.orchestra.run.trial151.stdout:34359738372 2026-02-19T20:29:15.115 INFO:teuthology.orchestra.run.trial151.stdout:34359738371 2026-02-19T20:29:15.125 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.2 2026-02-19T20:29:15.125 DEBUG:teuthology.parallel:result is None 2026-02-19T20:29:15.129 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738371 for osd.0 2026-02-19T20:29:15.129 DEBUG:teuthology.parallel:result is None 2026-02-19T20:29:15.129 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-19T20:29:15.130 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-19T20:29:15.333 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:29:15.334 INFO:teuthology.orchestra.run.trial151.stderr:dumped all 2026-02-19T20:29:15.344 INFO:teuthology.orchestra.run.trial151.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-19T20:29:14.454116+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82784,"kb_used_data":1528,"kb_used_omap":23,"kb_used_meta":80808,"kb_avail":283032736,"statfs":{"total":289910292480,"available":289825521664,"internally_reserved":0,"allocated":1564672,"data_stored":1287022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23725,"internal_metadata":82748243},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":11,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.710086"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.745596+0000","last_change":"2026-02-19T20:29:11.745770+0000","last_active":"2026-02-19T20:29:11.745596+0000","last_peered":"2026-02-19T20:29:11.745596+0000","last_clean":"2026-02-19T20:29:11.745596+0000","last_became_active":"2026-02-19T20:29:09.741410+0000","last_became_peered":"2026-02-19T20:29:09.741410+0000","last_unstale":"2026-02-19T20:29:11.745596+0000","last_undegraded":"2026-02-19T20:29:11.745596+0000","last_fullsized":"2026-02-19T20:29:11.745596+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:27:41.302717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052313100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.745756+0000","last_change":"2026-02-19T20:29:11.745905+0000","last_active":"2026-02-19T20:29:11.745756+0000","last_peered":"2026-02-19T20:29:11.745756+0000","last_clean":"2026-02-19T20:29:11.745756+0000","last_became_active":"2026-02-19T20:29:09.741207+0000","last_became_peered":"2026-02-19T20:29:09.741207+0000","last_unstale":"2026-02-19T20:29:11.745756+0000","last_undegraded":"2026-02-19T20:29:11.745756+0000","last_fullsized":"2026-02-19T20:29:11.745756+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:30:59.713739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066944300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.745569+0000","last_change":"2026-02-19T20:29:11.745724+0000","last_active":"2026-02-19T20:29:11.745569+0000","last_peered":"2026-02-19T20:29:11.745569+0000","last_clean":"2026-02-19T20:29:11.745569+0000","last_became_active":"2026-02-19T20:29:09.741235+0000","last_became_peered":"2026-02-19T20:29:09.741235+0000","last_unstale":"2026-02-19T20:29:11.745569+0000","last_undegraded":"2026-02-19T20:29:11.745569+0000","last_fullsized":"2026-02-19T20:29:11.745569+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:34:53.768773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051114900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.745395+0000","last_change":"2026-02-19T20:29:11.745539+0000","last_active":"2026-02-19T20:29:11.745395+0000","last_peered":"2026-02-19T20:29:11.745395+0000","last_clean":"2026-02-19T20:29:11.745395+0000","last_became_active":"2026-02-19T20:29:09.740922+0000","last_became_peered":"2026-02-19T20:29:09.740922+0000","last_unstale":"2026-02-19T20:29:11.745395+0000","last_undegraded":"2026-02-19T20:29:11.745395+0000","last_fullsized":"2026-02-19T20:29:11.745395+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:44:16.698249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353877,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.743941+0000","last_change":"2026-02-19T20:29:11.743941+0000","last_active":"2026-02-19T20:29:11.743941+0000","last_peered":"2026-02-19T20:29:11.743941+0000","last_clean":"2026-02-19T20:29:11.743941+0000","last_became_active":"2026-02-19T20:29:09.740605+0000","last_became_peered":"2026-02-19T20:29:09.740605+0000","last_unstale":"2026-02-19T20:29:11.743941+0000","last_undegraded":"2026-02-19T20:29:11.743941+0000","last_fullsized":"2026-02-19T20:29:11.743941+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T00:12:21.879309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083745499999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.741027+0000","last_change":"2026-02-19T20:29:11.741215+0000","last_active":"2026-02-19T20:29:11.741027+0000","last_peered":"2026-02-19T20:29:11.741027+0000","last_clean":"2026-02-19T20:29:11.741027+0000","last_became_active":"2026-02-19T20:29:09.740655+0000","last_became_peered":"2026-02-19T20:29:09.740655+0000","last_unstale":"2026-02-19T20:29:11.741027+0000","last_undegraded":"2026-02-19T20:29:11.741027+0000","last_fullsized":"2026-02-19T20:29:11.741027+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:52:18.535895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037179999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.741098+0000","last_change":"2026-02-19T20:29:11.741323+0000","last_active":"2026-02-19T20:29:11.741098+0000","last_peered":"2026-02-19T20:29:11.741098+0000","last_clean":"2026-02-19T20:29:11.741098+0000","last_became_active":"2026-02-19T20:29:09.740330+0000","last_became_peered":"2026-02-19T20:29:09.740330+0000","last_unstale":"2026-02-19T20:29:11.741098+0000","last_undegraded":"2026-02-19T20:29:11.741098+0000","last_fullsized":"2026-02-19T20:29:11.741098+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:54:37.429972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041288700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.745432+0000","last_change":"2026-02-19T20:29:11.745555+0000","last_active":"2026-02-19T20:29:11.745432+0000","last_peered":"2026-02-19T20:29:11.745432+0000","last_clean":"2026-02-19T20:29:11.745432+0000","last_became_active":"2026-02-19T20:29:09.740070+0000","last_became_peered":"2026-02-19T20:29:09.740070+0000","last_unstale":"2026-02-19T20:29:11.745432+0000","last_undegraded":"2026-02-19T20:29:11.745432+0000","last_fullsized":"2026-02-19T20:29:11.745432+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T22:03:17.974819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036770299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.744919+0000","last_change":"2026-02-19T20:29:07.943283+0000","last_active":"2026-02-19T20:29:11.744919+0000","last_peered":"2026-02-19T20:29:11.744919+0000","last_clean":"2026-02-19T20:29:11.744919+0000","last_became_active":"2026-02-19T20:29:07.942262+0000","last_became_peered":"2026-02-19T20:29:07.942262+0000","last_unstale":"2026-02-19T20:29:11.744919+0000","last_undegraded":"2026-02-19T20:29:11.744919+0000","last_fullsized":"2026-02-19T20:29:11.744919+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_clean_scrub_stamp":"2026-02-19T20:29:06.726163+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:37:51.447976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":122880,"data_stored":35416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27852,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94343988,"statfs":{"total":96636764160,"available":96608243712,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27852,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94343988,"statfs":{"total":96636764160,"available":96608243712,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T20:29:15.345 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-19T20:29:15.496 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:29:15.496 INFO:teuthology.orchestra.run.trial151.stderr:dumped all 2026-02-19T20:29:15.508 INFO:teuthology.orchestra.run.trial151.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-19T20:29:14.454116+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82784,"kb_used_data":1528,"kb_used_omap":23,"kb_used_meta":80808,"kb_avail":283032736,"statfs":{"total":289910292480,"available":289825521664,"internally_reserved":0,"allocated":1564672,"data_stored":1287022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23725,"internal_metadata":82748243},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":11,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.710086"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.745596+0000","last_change":"2026-02-19T20:29:11.745770+0000","last_active":"2026-02-19T20:29:11.745596+0000","last_peered":"2026-02-19T20:29:11.745596+0000","last_clean":"2026-02-19T20:29:11.745596+0000","last_became_active":"2026-02-19T20:29:09.741410+0000","last_became_peered":"2026-02-19T20:29:09.741410+0000","last_unstale":"2026-02-19T20:29:11.745596+0000","last_undegraded":"2026-02-19T20:29:11.745596+0000","last_fullsized":"2026-02-19T20:29:11.745596+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:27:41.302717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052313100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.745756+0000","last_change":"2026-02-19T20:29:11.745905+0000","last_active":"2026-02-19T20:29:11.745756+0000","last_peered":"2026-02-19T20:29:11.745756+0000","last_clean":"2026-02-19T20:29:11.745756+0000","last_became_active":"2026-02-19T20:29:09.741207+0000","last_became_peered":"2026-02-19T20:29:09.741207+0000","last_unstale":"2026-02-19T20:29:11.745756+0000","last_undegraded":"2026-02-19T20:29:11.745756+0000","last_fullsized":"2026-02-19T20:29:11.745756+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:30:59.713739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066944300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.745569+0000","last_change":"2026-02-19T20:29:11.745724+0000","last_active":"2026-02-19T20:29:11.745569+0000","last_peered":"2026-02-19T20:29:11.745569+0000","last_clean":"2026-02-19T20:29:11.745569+0000","last_became_active":"2026-02-19T20:29:09.741235+0000","last_became_peered":"2026-02-19T20:29:09.741235+0000","last_unstale":"2026-02-19T20:29:11.745569+0000","last_undegraded":"2026-02-19T20:29:11.745569+0000","last_fullsized":"2026-02-19T20:29:11.745569+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:34:53.768773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051114900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.745395+0000","last_change":"2026-02-19T20:29:11.745539+0000","last_active":"2026-02-19T20:29:11.745395+0000","last_peered":"2026-02-19T20:29:11.745395+0000","last_clean":"2026-02-19T20:29:11.745395+0000","last_became_active":"2026-02-19T20:29:09.740922+0000","last_became_peered":"2026-02-19T20:29:09.740922+0000","last_unstale":"2026-02-19T20:29:11.745395+0000","last_undegraded":"2026-02-19T20:29:11.745395+0000","last_fullsized":"2026-02-19T20:29:11.745395+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:44:16.698249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353877,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.743941+0000","last_change":"2026-02-19T20:29:11.743941+0000","last_active":"2026-02-19T20:29:11.743941+0000","last_peered":"2026-02-19T20:29:11.743941+0000","last_clean":"2026-02-19T20:29:11.743941+0000","last_became_active":"2026-02-19T20:29:09.740605+0000","last_became_peered":"2026-02-19T20:29:09.740605+0000","last_unstale":"2026-02-19T20:29:11.743941+0000","last_undegraded":"2026-02-19T20:29:11.743941+0000","last_fullsized":"2026-02-19T20:29:11.743941+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T00:12:21.879309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083745499999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.741027+0000","last_change":"2026-02-19T20:29:11.741215+0000","last_active":"2026-02-19T20:29:11.741027+0000","last_peered":"2026-02-19T20:29:11.741027+0000","last_clean":"2026-02-19T20:29:11.741027+0000","last_became_active":"2026-02-19T20:29:09.740655+0000","last_became_peered":"2026-02-19T20:29:09.740655+0000","last_unstale":"2026-02-19T20:29:11.741027+0000","last_undegraded":"2026-02-19T20:29:11.741027+0000","last_fullsized":"2026-02-19T20:29:11.741027+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:52:18.535895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037179999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.741098+0000","last_change":"2026-02-19T20:29:11.741323+0000","last_active":"2026-02-19T20:29:11.741098+0000","last_peered":"2026-02-19T20:29:11.741098+0000","last_clean":"2026-02-19T20:29:11.741098+0000","last_became_active":"2026-02-19T20:29:09.740330+0000","last_became_peered":"2026-02-19T20:29:09.740330+0000","last_unstale":"2026-02-19T20:29:11.741098+0000","last_undegraded":"2026-02-19T20:29:11.741098+0000","last_fullsized":"2026-02-19T20:29:11.741098+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:54:37.429972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041288700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.745432+0000","last_change":"2026-02-19T20:29:11.745555+0000","last_active":"2026-02-19T20:29:11.745432+0000","last_peered":"2026-02-19T20:29:11.745432+0000","last_clean":"2026-02-19T20:29:11.745432+0000","last_became_active":"2026-02-19T20:29:09.740070+0000","last_became_peered":"2026-02-19T20:29:09.740070+0000","last_unstale":"2026-02-19T20:29:11.745432+0000","last_undegraded":"2026-02-19T20:29:11.745432+0000","last_fullsized":"2026-02-19T20:29:11.745432+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T22:03:17.974819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036770299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-19T20:29:11.744919+0000","last_change":"2026-02-19T20:29:07.943283+0000","last_active":"2026-02-19T20:29:11.744919+0000","last_peered":"2026-02-19T20:29:11.744919+0000","last_clean":"2026-02-19T20:29:11.744919+0000","last_became_active":"2026-02-19T20:29:07.942262+0000","last_became_peered":"2026-02-19T20:29:07.942262+0000","last_unstale":"2026-02-19T20:29:11.744919+0000","last_undegraded":"2026-02-19T20:29:11.744919+0000","last_fullsized":"2026-02-19T20:29:11.744919+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_clean_scrub_stamp":"2026-02-19T20:29:06.726163+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:37:51.447976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":122880,"data_stored":35416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27852,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94343988,"statfs":{"total":96636764160,"available":96608243712,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27852,"kb_used_data":704,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94343988,"statfs":{"total":96636764160,"available":96608243712,"internally_reserved":0,"allocated":720896,"data_stored":625803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T20:29:15.509 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-19T20:29:15.509 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-19T20:29:15.510 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-19T20:29:15.510 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-19T20:29:15.691 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T20:29:15.691 INFO:teuthology.orchestra.run.trial151.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-19T20:29:15.703 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-19T20:29:15.703 INFO:teuthology.run_tasks:Running task exec... 2026-02-19T20:29:15.716 INFO:teuthology.task.exec:Executing custom commands... 2026-02-19T20:29:15.716 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial151.front.sepia.ceph.com 2026-02-19T20:29:15.716 DEBUG:teuthology.orchestra.run.trial151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-19T20:29:15.914 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror] 2026-02-19T20:29:15.914 INFO:teuthology.orchestra.run.trial151.stdout: key = AQBvcpdpwiy/AhAALN36hFPTJlLpRD7X4t1yWw== 2026-02-19T20:29:15.914 INFO:teuthology.orchestra.run.trial151.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-19T20:29:15.914 INFO:teuthology.orchestra.run.trial151.stdout: caps osd = "profile rbd" 2026-02-19T20:29:15.914 INFO:teuthology.orchestra.run.trial151.stderr:updated caps for client.mirror 2026-02-19T20:29:15.927 DEBUG:teuthology.orchestra.run.trial151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-19T20:29:16.121 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.0] 2026-02-19T20:29:16.122 INFO:teuthology.orchestra.run.trial151.stdout: key = AQBvcpdp29hHBxAA75mE77Lrx+T56ICXRBj7vg== 2026-02-19T20:29:16.122 INFO:teuthology.orchestra.run.trial151.stdout: caps mon = "profile rbd-mirror" 2026-02-19T20:29:16.122 INFO:teuthology.orchestra.run.trial151.stdout: caps osd = "profile rbd" 2026-02-19T20:29:16.122 INFO:teuthology.orchestra.run.trial151.stderr:updated caps for client.mirror.0 2026-02-19T20:29:16.133 DEBUG:teuthology.orchestra.run.trial151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-19T20:29:16.321 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.1] 2026-02-19T20:29:16.321 INFO:teuthology.orchestra.run.trial151.stdout: key = AQBvcpdpxZ6PCxAAaCu0jY7IYUPl8SiplorTjQ== 2026-02-19T20:29:16.321 INFO:teuthology.orchestra.run.trial151.stdout: caps mon = "profile rbd-mirror" 2026-02-19T20:29:16.322 INFO:teuthology.orchestra.run.trial151.stdout: caps osd = "profile rbd" 2026-02-19T20:29:16.322 INFO:teuthology.orchestra.run.trial151.stderr:updated caps for client.mirror.1 2026-02-19T20:29:16.335 DEBUG:teuthology.orchestra.run.trial151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-19T20:29:16.524 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.2] 2026-02-19T20:29:16.525 INFO:teuthology.orchestra.run.trial151.stdout: key = AQBvcpdpdC/VDxAAcLmHLrLf1OLTa+cbt7DEng== 2026-02-19T20:29:16.525 INFO:teuthology.orchestra.run.trial151.stdout: caps mon = "profile rbd-mirror" 2026-02-19T20:29:16.525 INFO:teuthology.orchestra.run.trial151.stdout: caps osd = "profile rbd" 2026-02-19T20:29:16.525 INFO:teuthology.orchestra.run.trial151.stderr:updated caps for client.mirror.2 2026-02-19T20:29:16.536 DEBUG:teuthology.orchestra.run.trial151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-19T20:29:16.728 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.3] 2026-02-19T20:29:16.728 INFO:teuthology.orchestra.run.trial151.stdout: key = AQBvcpdpFFEcFBAAJDHP4h0TREmJhMsVHEHq3w== 2026-02-19T20:29:16.728 INFO:teuthology.orchestra.run.trial151.stdout: caps mon = "profile rbd-mirror" 2026-02-19T20:29:16.728 INFO:teuthology.orchestra.run.trial151.stdout: caps osd = "profile rbd" 2026-02-19T20:29:16.728 INFO:teuthology.orchestra.run.trial151.stderr:updated caps for client.mirror.3 2026-02-19T20:29:16.738 DEBUG:teuthology.orchestra.run.trial151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-19T20:29:16.930 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror] 2026-02-19T20:29:16.930 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdp1PcxARAAn8bScqLX1zuDJCN57TxoIQ== 2026-02-19T20:29:16.930 INFO:teuthology.orchestra.run.trial151.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-19T20:29:16.930 INFO:teuthology.orchestra.run.trial151.stdout: caps osd = "profile rbd" 2026-02-19T20:29:16.931 INFO:teuthology.orchestra.run.trial151.stderr:updated caps for client.mirror 2026-02-19T20:29:16.944 DEBUG:teuthology.orchestra.run.trial151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-19T20:29:17.136 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.0] 2026-02-19T20:29:17.136 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpUTKSBRAAQ0nzucVNeILjwRb+UzVA2g== 2026-02-19T20:29:17.136 INFO:teuthology.orchestra.run.trial151.stdout: caps mon = "profile rbd-mirror" 2026-02-19T20:29:17.136 INFO:teuthology.orchestra.run.trial151.stdout: caps osd = "profile rbd" 2026-02-19T20:29:17.137 INFO:teuthology.orchestra.run.trial151.stderr:updated caps for client.mirror.0 2026-02-19T20:29:17.148 DEBUG:teuthology.orchestra.run.trial151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-19T20:29:17.345 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.1] 2026-02-19T20:29:17.346 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpI4UYChAAFYcC7W3z6tQR0h+ffSxRgQ== 2026-02-19T20:29:17.346 INFO:teuthology.orchestra.run.trial151.stdout: caps mon = "profile rbd-mirror" 2026-02-19T20:29:17.346 INFO:teuthology.orchestra.run.trial151.stdout: caps osd = "profile rbd" 2026-02-19T20:29:17.346 INFO:teuthology.orchestra.run.trial151.stderr:updated caps for client.mirror.1 2026-02-19T20:29:17.359 DEBUG:teuthology.orchestra.run.trial151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-19T20:29:17.550 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.2] 2026-02-19T20:29:17.551 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpgAAHDhAA74fPazFQufybPeEnyUWyyg== 2026-02-19T20:29:17.551 INFO:teuthology.orchestra.run.trial151.stdout: caps mon = "profile rbd-mirror" 2026-02-19T20:29:17.551 INFO:teuthology.orchestra.run.trial151.stdout: caps osd = "profile rbd" 2026-02-19T20:29:17.551 INFO:teuthology.orchestra.run.trial151.stderr:updated caps for client.mirror.2 2026-02-19T20:29:17.562 DEBUG:teuthology.orchestra.run.trial151:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-19T20:29:17.755 INFO:teuthology.orchestra.run.trial151.stdout:[client.mirror.3] 2026-02-19T20:29:17.756 INFO:teuthology.orchestra.run.trial151.stdout: key = AQCFcpdpOcpuEhAALg8CtOV+umSUh/LreuYuwA== 2026-02-19T20:29:17.756 INFO:teuthology.orchestra.run.trial151.stdout: caps mon = "profile rbd-mirror" 2026-02-19T20:29:17.756 INFO:teuthology.orchestra.run.trial151.stdout: caps osd = "profile rbd" 2026-02-19T20:29:17.756 INFO:teuthology.orchestra.run.trial151.stderr:updated caps for client.mirror.3 2026-02-19T20:29:17.767 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-19T20:29:17.782 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T20:29:17.782 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T20:29:17.784 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T20:29:17.784 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-19T20:29:17.799 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-19T20:29:17.799 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-19T20:29:17.801 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-19T20:29:17.801 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-19T20:29:17.816 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-19T20:29:17.816 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-19T20:29:17.818 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-19T20:29:17.819 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-19T20:29:17.833 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-19T20:29:17.834 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-19T20:29:17.835 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-19T20:29:17.836 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-19T20:29:17.851 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-19T20:29:17.851 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-19T20:29:17.853 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-19T20:29:17.853 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-19T20:29:17.912 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-19T20:29:17.912 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-19T20:29:17.914 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-19T20:29:17.914 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-19T20:29:17.925 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-19T20:29:17.925 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-19T20:29:17.927 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-19T20:29:17.927 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-19T20:29:17.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-19T20:29:17.939 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-19T20:29:17.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-19T20:29:17.940 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-19T20:29:17.959 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771532957 2026-02-19T20:29:17.959 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-19T20:29:17.959 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-19T20:29:17.971 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771532957 2026-02-19T20:29:17.971 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-19T20:29:17.971 INFO:teuthology.run_tasks:Running task workunit... 2026-02-19T20:29:17.989 INFO:tasks.workunit:Pulling workunits from ref 802bf111cd55e57f5842153d7ba2b8b964aa818a 2026-02-19T20:29:17.989 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-19T20:29:17.990 DEBUG:teuthology.orchestra.run.trial151:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-19T20:29:17.990 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2026-02-19T20:29:17.990 INFO:tasks.rbd_mirror_thrash:waiting for 118.5913637369585 secs before thrashing 2026-02-19T20:29:17.990 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2026-02-19T20:29:17.990 INFO:tasks.rbd_mirror_thrash:waiting for 76.25711278954341 secs before thrashing 2026-02-19T20:29:17.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T20:29:17.993 INFO:teuthology.orchestra.run.trial151.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-19T20:29:17.994 DEBUG:teuthology.orchestra.run.trial151:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-19T20:29:18.040 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-19T20:29:18.041 DEBUG:teuthology.orchestra.run.trial151:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-19T20:29:18.084 INFO:tasks.workunit:timeout=6h 2026-02-19T20:29:18.084 INFO:tasks.workunit:cleanup=True 2026-02-19T20:29:18.085 DEBUG:teuthology.orchestra.run.trial151:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout 802bf111cd55e57f5842153d7ba2b8b964aa818a 2026-02-19T20:29:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-19T20:29:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Note: switching to '802bf111cd55e57f5842153d7ba2b8b964aa818a'. 2026-02-19T20:29:40.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: 2026-02-19T20:29:40.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-19T20:29:40.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-19T20:29:40.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:state without impacting any branches by switching back to a branch. 2026-02-19T20:29:40.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: 2026-02-19T20:29:40.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-19T20:29:40.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-19T20:29:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: 2026-02-19T20:29:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: git switch -c 2026-02-19T20:29:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: 2026-02-19T20:29:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Or undo this operation with: 2026-02-19T20:29:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: 2026-02-19T20:29:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: git switch - 2026-02-19T20:29:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: 2026-02-19T20:29:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-19T20:29:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: 2026-02-19T20:29:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:HEAD is now at 802bf111cd5 Merge branch 'wip-common-ceph_json-jfw-spirit-to-boost' of https://github.com/chardan/ceph into wip-yuri-testing-2026-02-18-1820 2026-02-19T20:29:40.616 DEBUG:teuthology.orchestra.run.trial151:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-19T20:29:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-19T20:29:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-19T20:29:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-19T20:29:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-19T20:29:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-19T20:29:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-19T20:29:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-19T20:29:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-19T20:29:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-19T20:29:40.844 DEBUG:teuthology.orchestra.run.trial151:> set -ex 2026-02-19T20:29:40.844 DEBUG:teuthology.orchestra.run.trial151:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-19T20:29:40.890 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2026-02-19T20:29:40.893 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2026-02-19T20:29:40.893 DEBUG:teuthology.orchestra.run.trial151:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802bf111cd55e57f5842153d7ba2b8b964aa818a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=pool RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=journal RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-19T20:29:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ IMAGE_COUNT=50 2026-02-19T20:29:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ export LOCKDEP=0 2026-02-19T20:29:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ LOCKDEP=0 2026-02-19T20:29:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-19T20:29:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-19T20:29:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ RBD_MIRROR_INSTANCES=4 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CLUSTER1=cluster1 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CLUSTER2=cluster2 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ POOL=mirror 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ PARENT_POOL=mirror_parent 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ NS1=ns1 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ NS2=ns2 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ TEMPDIR= 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ID=mirror 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,journaling 2026-02-19T20:29:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-19T20:29:40.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ RBD_MIRROR_MODE=journal 2026-02-19T20:29:40.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ MIRROR_POOL_MODE=pool 2026-02-19T20:29:40.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' journal = snapshot ']' 2026-02-19T20:29:40.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-19T20:29:40.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS='--id mirror' 2026-02-19T20:29:40.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ LAST_MIRROR_INSTANCE=3 2026-02-19T20:29:40.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-19T20:29:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-19T20:29:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-19T20:29:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_BIN=. 2026-02-19T20:29:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_SRC=. 2026-02-19T20:29:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-19T20:29:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 -gt 0 ']' 2026-02-19T20:29:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ setup 2026-02-19T20:29:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local c 2026-02-19T20:29:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-19T20:29:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ setup_tempdir 2026-02-19T20:29:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ mktemp -d 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ TEMPDIR=/tmp/tmp.xpxdoJdHT7 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -z 1 ']' 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ setup_pools cluster1 cluster2 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_cluster=cluster2 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mon_map_file 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mon_addr 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local admin_key_file 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local uuid 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T20:29:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-19T20:29:41.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:pool 'mirror' already exists 2026-02-19T20:29:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T20:29:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-19T20:29:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:pool 'mirror_parent' already exists 2026-02-19T20:29:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T20:29:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-19T20:29:45.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T20:29:45.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-19T20:29:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T20:29:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror pool 2026-02-19T20:29:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-19T20:29:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-19T20:29:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-19T20:29:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-19T20:29:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 pool 2026-02-19T20:29:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-19T20:29:48.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 pool 2026-02-19T20:29:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -z ']' 2026-02-19T20:29:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -z ']' 2026-02-19T20:29:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ peer_add cluster1 mirror cluster2 2026-02-19T20:29:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local client_cluster=cluster2 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_cluster=cluster2 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local uuid_var_name 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local error_code 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local peer_uuid 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 16 32 2026-02-19T20:29:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:29:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2026-02-19T20:29:49.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ peer_uuid=62303cdb-5bd2-495c-b8d8-bc350b406aa3 2026-02-19T20:29:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=0 2026-02-19T20:29:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:29:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 17 ']' 2026-02-19T20:29:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 -eq 0 2026-02-19T20:29:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:29:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ peer_add cluster1 mirror_parent cluster2 2026-02-19T20:29:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:29:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:49.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror_parent 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local client_cluster=cluster2 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_cluster=cluster2 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local uuid_var_name 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local error_code 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local peer_uuid 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 16 32 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:29:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2026-02-19T20:29:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ peer_uuid=ffbd140e-fc50-4d2b-866b-7095b394633b 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=0 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 17 ']' 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 -eq 0 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ setup_pools cluster2 cluster1 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_cluster=cluster1 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mon_map_file 2026-02-19T20:29:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mon_addr 2026-02-19T20:29:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local admin_key_file 2026-02-19T20:29:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local uuid 2026-02-19T20:29:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T20:29:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-19T20:29:49.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:pool 'mirror' already exists 2026-02-19T20:29:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T20:29:49.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-19T20:29:50.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:pool 'mirror_parent' already exists 2026-02-19T20:29:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T20:29:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-19T20:29:53.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T20:29:53.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-19T20:29:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T20:29:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror pool 2026-02-19T20:29:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-19T20:29:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-19T20:29:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-19T20:29:56.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-19T20:29:56.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 pool 2026-02-19T20:29:56.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-19T20:29:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 pool 2026-02-19T20:29:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -z ']' 2026-02-19T20:29:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -z ']' 2026-02-19T20:29:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ peer_add cluster2 mirror cluster1 2026-02-19T20:29:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:29:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local client_cluster=cluster1 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_cluster=cluster1 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local uuid_var_name 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local error_code 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local peer_uuid 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 16 32 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:29:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2026-02-19T20:29:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ peer_uuid=63a6310b-64c5-44db-b9f4-d0f0211d79bf 2026-02-19T20:29:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=0 2026-02-19T20:29:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:29:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 17 ']' 2026-02-19T20:29:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 -eq 0 2026-02-19T20:29:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ peer_add cluster2 mirror_parent cluster1 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror_parent 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local client_cluster=cluster1 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_cluster=cluster1 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local uuid_var_name 2026-02-19T20:29:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local error_code 2026-02-19T20:29:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local peer_uuid 2026-02-19T20:29:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 16 32 2026-02-19T20:29:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:29:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2026-02-19T20:29:56.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ peer_uuid=f8c49d91-f5e2-4fa4-9299-a73cdd19adb8 2026-02-19T20:29:56.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=0 2026-02-19T20:29:56.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:29:56.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 17 ']' 2026-02-19T20:29:56.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 -eq 0 2026-02-19T20:29:56.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:56.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:29:56.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:29:56.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirrors cluster1 2026-02-19T20:29:56.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:29:56.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:29:56.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:56.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirror cluster1:0 2026-02-19T20:29:56.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:56.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirror cluster1:1 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:29:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirror cluster1:2 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:29:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:29:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:29:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:29:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:29:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:29:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:29:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:29:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:29:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:29:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirror cluster1:3 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirrors cluster2 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirror cluster2:0 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2:0 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster2:0 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster2 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-19T20:29:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster2 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster2 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirror cluster2:1 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2:1 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster2:1 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster2 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster2 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster2 2026-02-19T20:29:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirror cluster2:2 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2:2 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster2:2 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster2 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster2 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster2 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T20:29:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirror cluster2:3 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2:3 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster2:3 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster2 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster2 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster2 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ tee -a /tmp/tmp.xpxdoJdHT7/rbd-mirror.test.log 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ date '+%F %T' 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo 2026-02-19 20:29:56 TEST: add image and test replay after client crashes 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:2026-02-19 20:29:56 TEST: add image and test replay after client crashes 2026-02-19T20:29:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=test 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror test journal 512M 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror test 512M 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:29:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/test 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:29:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:29:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:29:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:29:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:29:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:29:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:29:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:29:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:29:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:29:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:29:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:29:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:29:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:29:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:29:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:29:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:29:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:29:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:29:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:29:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:29:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:29:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:29:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:29:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:29:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:29:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:29:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:29:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:29:57.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:29:57.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:29:57.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:29:57.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:29:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:29:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:29:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:29:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:29:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:29:57.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:29:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:29:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:29:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:29:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:29:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:29:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:29:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-19T20:29:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:29:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:29:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:29:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:29:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:29:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:29:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:29:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:29:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:29:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:29:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:29:58.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:29:58.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-19T20:29:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:29:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:29:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:29:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-19T20:29:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:29:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:29:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:29:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:29:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:29:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:29:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:29:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:29:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:29:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:29:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:29:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:29:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:29:58.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:29:58.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:29:58.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:29:58.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:29:58.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:29:58.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:29:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:29:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:29:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:29:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:29:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:29:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:29:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:29:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:29:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:29:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:29:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:29:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-19T20:29:58.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:29:58.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:29:58.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:29:58.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:29:58.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 1 2026-02-19T20:29:59.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:29:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:29:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:29:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:29:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:29:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:29:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:29:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:29:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:29:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:29:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:29:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:29:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:29:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:29:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:29:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:29:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:29:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:29:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:29:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:29:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:29:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:29:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:29:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:29:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:29:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:29:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:29:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:29:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:29:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:29:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:29:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:29:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:29:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:29:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:29:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:29:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:29:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:29:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:29:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:29:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:29:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:29:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:29:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:29:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:29:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:29:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:29:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:29:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:29:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:29:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:29:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:29:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:29:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:29:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:29:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:29:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:29:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:29:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:29:59.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:29:59.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:29:59.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:29:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-19T20:29:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:29:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:29:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:29:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:29:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:29:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:29:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:29:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:29:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:29:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:29:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:29:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:29:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:29:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:29:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:29:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:29:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:29:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-19T20:29:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:29:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:29:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:29:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:29:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-19T20:29:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:29:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:29:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:29:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:29:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:29:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:29:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:29:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:29:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:29:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:29:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:29:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-19T20:29:59.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:29:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:29:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:29:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:29:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 2 2026-02-19T20:30:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:30:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:30:01.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:30:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:01.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:30:01.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:30:01.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:01.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:01.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:30:01.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:30:01.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:30:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:01.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:30:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:30:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:30:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:30:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:30:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:30:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:30:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:30:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:30:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:30:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:30:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:30:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:30:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:30:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:30:01.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:30:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-19T20:30:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:30:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:30:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:30:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:30:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:30:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:30:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:30:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:30:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-19T20:30:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-19T20:30:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:30:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:30:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:30:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:30:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:30:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:30:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:30:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:30:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-19T20:30:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:30:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 4 2026-02-19T20:30:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:30:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:30:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:30:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:30:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:30:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:30:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:30:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:30:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:30:05.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:05.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:05.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:05.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:30:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:30:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:30:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:30:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:30:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:30:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:30:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:30:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:30:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:30:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:30:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:30:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:30:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:30:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:30:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:30:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:30:05.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:30:05.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:30:05.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:30:05.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-19T20:30:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:05.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:05.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:30:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:05.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:30:05.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:30:05.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:05.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:05.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:30:05.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:30:05.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:30:05.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:30:05.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:30:05.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:30:05.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-19T20:30:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:30:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 8 2026-02-19T20:30:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:30:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:30:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:30:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:30:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:30:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:30:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:30:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:30:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:30:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:30:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:30:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:30:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:30:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:30:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:30:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:30:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:30:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:30:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:30:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:30:14.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:30:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:30:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:30:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:30:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-19T20:30:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:30:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:30:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:30:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:30:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:30:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:30:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:30:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:30:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:30:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:30:14.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:30:14.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:30:14.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:30:14.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-19T20:30:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:14.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:14.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-19T20:30:14.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:30:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:30:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:30:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:30:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:30:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:30:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:30:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:30:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-19T20:30:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:30:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:30:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:30:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:30:14.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 8 2026-02-19T20:30:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:30:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:30:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:30:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:30:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:30:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:30:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:30:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:30:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:30:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:30:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:30:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:30:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:30:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:30:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:30:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:30:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:30:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:30:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:30:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:30:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:30:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:30:22.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:22.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:22.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:30:22.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:30:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:30:22.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:30:22.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:30:22.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:30:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:30:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:30:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:30:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:30:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name= 2026-02-19T20:30:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 10 2026-02-19T20:30:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:30:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror test 2026-02-19T20:30:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:30:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:30:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:30:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:30:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=7 2026-02-19T20:30:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:30:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-19T20:30:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 633856 bytes 2026-02-19T20:30:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 54155264 bytes 2026-02-19T20:30:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 75572224 bytes 2026-02-19T20:30:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 173408256 bytes 2026-02-19T20:30:24.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 187702272 bytes 2026-02-19T20:30:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 254579712 bytes 2026-02-19T20:30:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 258769920 bytes 2026-02-19T20:30:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 274400256 bytes 2026-02-19T20:30:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 301806592 bytes 2026-02-19T20:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 350871552 bytes 2026-02-19T20:30:26.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 414842880 bytes 2026-02-19T20:30:27.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 487061504 bytes 2026-02-19T20:30:27.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 510404608 bytes 2026-02-19T20:30:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 532357120 bytes 2026-02-19T20:30:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 588546048 bytes 2026-02-19T20:30:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:30:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:30:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:30:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:30:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-19T20:30:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:30:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:30:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:30:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:30:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:30:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:30:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:30:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:30:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-19T20:30:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:30:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:30:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:30:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:30:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:30:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:30:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:30:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:30:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-19T20:30:30.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='test: 2026-02-19T20:30:30.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: global_id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:30:30.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+replaying 2026-02-19T20:30:30.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-19T20:30:30.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: service: mirror.0 on trial151 2026-02-19T20:30:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:30:21 2026-02-19T20:30:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: peer_sites: 2026-02-19T20:30:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: name: cluster2 2026-02-19T20:30:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: down+unknown 2026-02-19T20:30:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: status not found 2026-02-19T20:30:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: ' 2026-02-19T20:30:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ state: .*up+replaying' 2026-02-19T20:30:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: state: up+replaying 2026-02-19T20:30:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ description: .*' 2026-02-19T20:30:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-19T20:30:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:30:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:30:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^up+' 2026-02-19T20:30:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:30:30.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep 'service: *mirror..* on ' 2026-02-19T20:30:30.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: service: mirror.0 on trial151 2026-02-19T20:30:30.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update 2026-02-19T20:30:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-19T20:30:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:30:21' 2026-02-19T20:30:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-19 20:30:21' 2026-02-19T20:30:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:30:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:30:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:30:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:30:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'prev_last_update=2026-02-19 20:30:21' 2026-02-19T20:30:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:30:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:30:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-19T20:30:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='WARNINGOKWARNING142024259mirror.0trial15120.3.0-5336-g802bf111trueOK42064262mirror.1trial15120.3.0-5336-g802bf111falseOK42104271mirror.2trial15120.3.0-5336-g802bf111falseOK42144267mirror.3trial15120.3.0-5336-g802bf111falseOKtest959deb13-de1a-48d9-a341-1f95416f8379up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42024259mirror.0trial1512026-02-19 20:30:21cluster2a054b997-3cfb-48e6-978a-70d6a85e56dedown+unknownstatus not found' 2026-02-19T20:30:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update state 2026-02-19T20:30:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-19T20:30:30.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:30:21' 2026-02-19T20:30:30.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-19T20:30:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=up+replaying 2026-02-19T20:30:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:30:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep up+replaying 2026-02-19T20:30:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:30:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:30:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:30:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap1 2026-02-19T20:30:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap1 2026-02-19T20:30:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:30:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:30:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:30:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap1 2026-02-19T20:30:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:30:34.248 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3311389191607149) > weight (0.25) 2026-02-19T20:30:34.248 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8368183062254898) > weight (0.25) 2026-02-19T20:30:34.248 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.4354734189408751) > weight (0.25) 2026-02-19T20:30:34.249 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6737854951132831) > weight (0.25) 2026-02-19T20:30:34.249 INFO:tasks.rbd_mirror_thrash:waiting for 115.34681533914076 secs before thrashing 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:31:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:31:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:31:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:31:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:31:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:31:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:31:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:31:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:31:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:31:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:31:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:31:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:31:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:31:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:31:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:31:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:31:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:31:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:31:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:31:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:31:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:31:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:31:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:31:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:31:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:31:02.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:31:02.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:31:02.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:31:02.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:31:02.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:31:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:31:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:31:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:31:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:31:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:31:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:31:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:03.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:03.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:03.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:03.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=35303], [object_number=34, tag_tid=2, entry_tid=35302], [object_number=33, tag_tid=2, entry_tid=35301], [object_number=32, tag_tid=2, entry_tid=35300]]], state=connected] 2026-02-19T20:31:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:03.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:03.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=35303], [object_number=34, tag_tid=2, entry_tid=35302], [object_number=33, tag_tid=2, entry_tid=35301], [object_number=32, tag_tid=2, entry_tid=35300]]], state=connected] 2026-02-19T20:31:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=35, tag_tid=2, entry_tid=35303' 2026-02-19T20:31:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=35, tag_tid=2, entry_tid=35303' 2026-02-19T20:31:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=35303' '!=' '' 2026-02-19T20:31:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=35303' = '' 2026-02-19T20:31:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=35, tag_tid=2, entry_tid=35303' 2026-02-19T20:31:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=35303' 2026-02-19T20:31:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:03.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:03.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:03.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=35303' 2026-02-19T20:31:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35303 2026-02-19T20:31:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 35303 2026-02-19T20:31:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:03.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:03.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:03.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:03.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:03.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:03.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:03.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:03.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:03.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=35611], [object_number=34, tag_tid=2, entry_tid=35610], [object_number=33, tag_tid=2, entry_tid=35609], [object_number=32, tag_tid=2, entry_tid=35608]]], state=connected] 2026-02-19T20:31:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:03.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:03.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:03.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:03.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:03.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=35611], [object_number=34, tag_tid=2, entry_tid=35610], [object_number=33, tag_tid=2, entry_tid=35609], [object_number=32, tag_tid=2, entry_tid=35608]]], state=connected] 2026-02-19T20:31:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=35, tag_tid=2, entry_tid=35611' 2026-02-19T20:31:03.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=35, tag_tid=2, entry_tid=35611' 2026-02-19T20:31:03.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=35611' '!=' 'object_number=35, tag_tid=2, entry_tid=35303' 2026-02-19T20:31:03.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:03.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=35611' = 'object_number=35, tag_tid=2, entry_tid=35303' 2026-02-19T20:31:03.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=35, tag_tid=2, entry_tid=35611' 2026-02-19T20:31:03.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:03.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:03.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=35611' 2026-02-19T20:31:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=35611' 2026-02-19T20:31:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35611 2026-02-19T20:31:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 35611 2026-02-19T20:31:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:03.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:03.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:03.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:03.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:03.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:03.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:03.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:03.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:03.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:03.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:03.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:03.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:03.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:03.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=32, tag_tid=2, entry_tid=35900], [object_number=35, tag_tid=2, entry_tid=35899], [object_number=34, tag_tid=2, entry_tid=35898], [object_number=33, tag_tid=2, entry_tid=35897]]], state=connected] 2026-02-19T20:31:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:03.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:03.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=32, tag_tid=2, entry_tid=35900], [object_number=35, tag_tid=2, entry_tid=35899], [object_number=34, tag_tid=2, entry_tid=35898], [object_number=33, tag_tid=2, entry_tid=35897]]], state=connected] 2026-02-19T20:31:03.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=32, tag_tid=2, entry_tid=35900' 2026-02-19T20:31:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=32, tag_tid=2, entry_tid=35900' 2026-02-19T20:31:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=32, tag_tid=2, entry_tid=35900' '!=' 'object_number=35, tag_tid=2, entry_tid=35611' 2026-02-19T20:31:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=32, tag_tid=2, entry_tid=35900' = 'object_number=35, tag_tid=2, entry_tid=35611' 2026-02-19T20:31:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=32, tag_tid=2, entry_tid=35900' 2026-02-19T20:31:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:03.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:03.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:03.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=32, tag_tid=2, entry_tid=35900' 2026-02-19T20:31:03.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:03.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:03.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:03.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:03.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=32, tag_tid=2, entry_tid=35900' 2026-02-19T20:31:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:03.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35900 2026-02-19T20:31:03.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 35900 2026-02-19T20:31:03.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:03.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:03.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:04.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:04.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:04.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:04.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:04.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=2, entry_tid=36218], [object_number=33, tag_tid=2, entry_tid=36217], [object_number=32, tag_tid=2, entry_tid=36216], [object_number=35, tag_tid=2, entry_tid=36215]]], state=connected] 2026-02-19T20:31:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=2, entry_tid=36218], [object_number=33, tag_tid=2, entry_tid=36217], [object_number=32, tag_tid=2, entry_tid=36216], [object_number=35, tag_tid=2, entry_tid=36215]]], state=connected] 2026-02-19T20:31:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=2, entry_tid=36218' 2026-02-19T20:31:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=34, tag_tid=2, entry_tid=36218' 2026-02-19T20:31:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=2, entry_tid=36218' '!=' 'object_number=32, tag_tid=2, entry_tid=35900' 2026-02-19T20:31:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=2, entry_tid=36218' = 'object_number=32, tag_tid=2, entry_tid=35900' 2026-02-19T20:31:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=34, tag_tid=2, entry_tid=36218' 2026-02-19T20:31:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=2, entry_tid=36218' 2026-02-19T20:31:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=2, entry_tid=36218' 2026-02-19T20:31:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36218 2026-02-19T20:31:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 36218 2026-02-19T20:31:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:04.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:04.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=36575], [object_number=34, tag_tid=2, entry_tid=36574], [object_number=33, tag_tid=2, entry_tid=36573], [object_number=32, tag_tid=2, entry_tid=36572]]], state=connected] 2026-02-19T20:31:04.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=36575], [object_number=34, tag_tid=2, entry_tid=36574], [object_number=33, tag_tid=2, entry_tid=36573], [object_number=32, tag_tid=2, entry_tid=36572]]], state=connected] 2026-02-19T20:31:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=35, tag_tid=2, entry_tid=36575' 2026-02-19T20:31:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=35, tag_tid=2, entry_tid=36575' 2026-02-19T20:31:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=36575' '!=' 'object_number=34, tag_tid=2, entry_tid=36218' 2026-02-19T20:31:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=36575' = 'object_number=34, tag_tid=2, entry_tid=36218' 2026-02-19T20:31:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=35, tag_tid=2, entry_tid=36575' 2026-02-19T20:31:04.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:04.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:04.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:04.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=36575' 2026-02-19T20:31:04.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:04.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=36575' 2026-02-19T20:31:04.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:04.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36575 2026-02-19T20:31:04.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 36575 2026-02-19T20:31:04.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:04.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:04.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:04.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=36951], [object_number=34, tag_tid=2, entry_tid=36950], [object_number=33, tag_tid=2, entry_tid=36949], [object_number=32, tag_tid=2, entry_tid=36948]]], state=connected] 2026-02-19T20:31:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:04.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:04.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:04.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:04.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:04.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:04.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:04.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:04.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=36951], [object_number=34, tag_tid=2, entry_tid=36950], [object_number=33, tag_tid=2, entry_tid=36949], [object_number=32, tag_tid=2, entry_tid=36948]]], state=connected] 2026-02-19T20:31:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=35, tag_tid=2, entry_tid=36951' 2026-02-19T20:31:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=35, tag_tid=2, entry_tid=36951' 2026-02-19T20:31:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=36951' '!=' 'object_number=35, tag_tid=2, entry_tid=36575' 2026-02-19T20:31:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=36951' = 'object_number=35, tag_tid=2, entry_tid=36575' 2026-02-19T20:31:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=35, tag_tid=2, entry_tid=36951' 2026-02-19T20:31:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=36951' 2026-02-19T20:31:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:04.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=36951' 2026-02-19T20:31:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:04.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36951 2026-02-19T20:31:04.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 36951 2026-02-19T20:31:04.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:04.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:04.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:05.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=32, tag_tid=2, entry_tid=37264], [object_number=35, tag_tid=2, entry_tid=37263], [object_number=34, tag_tid=2, entry_tid=37262], [object_number=33, tag_tid=2, entry_tid=37261]]], state=connected] 2026-02-19T20:31:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:05.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:05.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:05.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:05.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:05.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:05.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=32, tag_tid=2, entry_tid=37264], [object_number=35, tag_tid=2, entry_tid=37263], [object_number=34, tag_tid=2, entry_tid=37262], [object_number=33, tag_tid=2, entry_tid=37261]]], state=connected] 2026-02-19T20:31:05.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=32, tag_tid=2, entry_tid=37264' 2026-02-19T20:31:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=32, tag_tid=2, entry_tid=37264' 2026-02-19T20:31:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=32, tag_tid=2, entry_tid=37264' '!=' 'object_number=35, tag_tid=2, entry_tid=36951' 2026-02-19T20:31:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=32, tag_tid=2, entry_tid=37264' = 'object_number=35, tag_tid=2, entry_tid=36951' 2026-02-19T20:31:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=32, tag_tid=2, entry_tid=37264' 2026-02-19T20:31:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:05.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:05.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=32, tag_tid=2, entry_tid=37264' 2026-02-19T20:31:05.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:05.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=32, tag_tid=2, entry_tid=37264' 2026-02-19T20:31:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37264 2026-02-19T20:31:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 37264 2026-02-19T20:31:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:05.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:05.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=37611], [object_number=34, tag_tid=2, entry_tid=37610], [object_number=33, tag_tid=2, entry_tid=37609], [object_number=32, tag_tid=2, entry_tid=37608]]], state=connected] 2026-02-19T20:31:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:31:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=37611], [object_number=34, tag_tid=2, entry_tid=37610], [object_number=33, tag_tid=2, entry_tid=37609], [object_number=32, tag_tid=2, entry_tid=37608]]], state=connected] 2026-02-19T20:31:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=35, tag_tid=2, entry_tid=37611' 2026-02-19T20:31:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=35, tag_tid=2, entry_tid=37611' 2026-02-19T20:31:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=37611' '!=' 'object_number=32, tag_tid=2, entry_tid=37264' 2026-02-19T20:31:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=37611' = 'object_number=32, tag_tid=2, entry_tid=37264' 2026-02-19T20:31:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=35, tag_tid=2, entry_tid=37611' 2026-02-19T20:31:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:05.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=37611' 2026-02-19T20:31:05.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:05.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=37611' 2026-02-19T20:31:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:05.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37611 2026-02-19T20:31:05.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 37611 2026-02-19T20:31:05.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:05.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:05.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:05.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:05.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:05.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:05.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:05.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:05.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=37852], [object_number=39, tag_tid=2, entry_tid=37851], [object_number=38, tag_tid=2, entry_tid=37850], [object_number=37, tag_tid=2, entry_tid=37849]]], state=connected] 2026-02-19T20:31:06.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:06.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=37852], [object_number=39, tag_tid=2, entry_tid=37851], [object_number=38, tag_tid=2, entry_tid=37850], [object_number=37, tag_tid=2, entry_tid=37849]]], state=connected] 2026-02-19T20:31:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=36, tag_tid=2, entry_tid=37852' 2026-02-19T20:31:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=36, tag_tid=2, entry_tid=37852' 2026-02-19T20:31:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=37852' '!=' 'object_number=35, tag_tid=2, entry_tid=37611' 2026-02-19T20:31:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=37852' = 'object_number=35, tag_tid=2, entry_tid=37611' 2026-02-19T20:31:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=36, tag_tid=2, entry_tid=37852' 2026-02-19T20:31:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:06.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=37852' 2026-02-19T20:31:06.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:06.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=37852' 2026-02-19T20:31:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:06.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37852 2026-02-19T20:31:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 37852 2026-02-19T20:31:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:06.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:06.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:06.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:06.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:06.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:06.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:06.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:06.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:06.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:06.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:06.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:06.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:06.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:06.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:06.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=38170], [object_number=37, tag_tid=2, entry_tid=38169], [object_number=36, tag_tid=2, entry_tid=38168], [object_number=39, tag_tid=2, entry_tid=38167]]], state=connected] 2026-02-19T20:31:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:06.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:06.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=38170], [object_number=37, tag_tid=2, entry_tid=38169], [object_number=36, tag_tid=2, entry_tid=38168], [object_number=39, tag_tid=2, entry_tid=38167]]], state=connected] 2026-02-19T20:31:06.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=2, entry_tid=38170' 2026-02-19T20:31:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=38, tag_tid=2, entry_tid=38170' 2026-02-19T20:31:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=38170' '!=' 'object_number=36, tag_tid=2, entry_tid=37852' 2026-02-19T20:31:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=38170' = 'object_number=36, tag_tid=2, entry_tid=37852' 2026-02-19T20:31:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=38, tag_tid=2, entry_tid=38170' 2026-02-19T20:31:06.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:06.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:06.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=38170' 2026-02-19T20:31:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=38170' 2026-02-19T20:31:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:06.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38170 2026-02-19T20:31:06.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 38170 2026-02-19T20:31:06.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:06.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:06.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:06.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:06.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:06.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:06.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:06.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:06.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:06.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:06.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=38451], [object_number=38, tag_tid=2, entry_tid=38450], [object_number=37, tag_tid=2, entry_tid=38449], [object_number=36, tag_tid=2, entry_tid=38448]]], state=connected] 2026-02-19T20:31:06.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=38451], [object_number=38, tag_tid=2, entry_tid=38450], [object_number=37, tag_tid=2, entry_tid=38449], [object_number=36, tag_tid=2, entry_tid=38448]]], state=connected] 2026-02-19T20:31:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=2, entry_tid=38451' 2026-02-19T20:31:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=39, tag_tid=2, entry_tid=38451' 2026-02-19T20:31:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=38451' '!=' 'object_number=38, tag_tid=2, entry_tid=38170' 2026-02-19T20:31:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=38451' = 'object_number=38, tag_tid=2, entry_tid=38170' 2026-02-19T20:31:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=39, tag_tid=2, entry_tid=38451' 2026-02-19T20:31:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=38451' 2026-02-19T20:31:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=38451' 2026-02-19T20:31:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38451 2026-02-19T20:31:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 38451 2026-02-19T20:31:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:06.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:06.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:06.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:06.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:07.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:07.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:07.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:07.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=38757], [object_number=36, tag_tid=2, entry_tid=38756], [object_number=39, tag_tid=2, entry_tid=38755], [object_number=38, tag_tid=2, entry_tid=38754]]], state=connected] 2026-02-19T20:31:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=38757], [object_number=36, tag_tid=2, entry_tid=38756], [object_number=39, tag_tid=2, entry_tid=38755], [object_number=38, tag_tid=2, entry_tid=38754]]], state=connected] 2026-02-19T20:31:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=38757' 2026-02-19T20:31:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=38757' 2026-02-19T20:31:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=38757' '!=' 'object_number=39, tag_tid=2, entry_tid=38451' 2026-02-19T20:31:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=38757' = 'object_number=39, tag_tid=2, entry_tid=38451' 2026-02-19T20:31:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=37, tag_tid=2, entry_tid=38757' 2026-02-19T20:31:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=38757' 2026-02-19T20:31:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=38757' 2026-02-19T20:31:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38757 2026-02-19T20:31:07.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 38757 2026-02-19T20:31:07.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:07.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:07.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:07.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:07.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:07.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:07.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:07.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:07.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:07.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:07.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:07.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:07.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:07.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:07.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:07.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:07.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:07.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:07.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:07.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:07.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:07.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:07.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:07.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:07.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:07.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:07.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=39071], [object_number=38, tag_tid=2, entry_tid=39070], [object_number=37, tag_tid=2, entry_tid=39069], [object_number=36, tag_tid=2, entry_tid=39068]]], state=connected] 2026-02-19T20:31:07.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=39071], [object_number=38, tag_tid=2, entry_tid=39070], [object_number=37, tag_tid=2, entry_tid=39069], [object_number=36, tag_tid=2, entry_tid=39068]]], state=connected] 2026-02-19T20:31:07.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=2, entry_tid=39071' 2026-02-19T20:31:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=39, tag_tid=2, entry_tid=39071' 2026-02-19T20:31:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=39071' '!=' 'object_number=37, tag_tid=2, entry_tid=38757' 2026-02-19T20:31:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=39071' = 'object_number=37, tag_tid=2, entry_tid=38757' 2026-02-19T20:31:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=39, tag_tid=2, entry_tid=39071' 2026-02-19T20:31:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:07.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=39071' 2026-02-19T20:31:07.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:07.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:07.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:07.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:07.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:07.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=39071' 2026-02-19T20:31:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39071 2026-02-19T20:31:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 39071 2026-02-19T20:31:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:07.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=39392], [object_number=39, tag_tid=2, entry_tid=39391], [object_number=38, tag_tid=2, entry_tid=39390], [object_number=37, tag_tid=2, entry_tid=39389]]], state=connected] 2026-02-19T20:31:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=39392], [object_number=39, tag_tid=2, entry_tid=39391], [object_number=38, tag_tid=2, entry_tid=39390], [object_number=37, tag_tid=2, entry_tid=39389]]], state=connected] 2026-02-19T20:31:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=36, tag_tid=2, entry_tid=39392' 2026-02-19T20:31:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=36, tag_tid=2, entry_tid=39392' 2026-02-19T20:31:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=39392' '!=' 'object_number=39, tag_tid=2, entry_tid=39071' 2026-02-19T20:31:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=39392' = 'object_number=39, tag_tid=2, entry_tid=39071' 2026-02-19T20:31:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=36, tag_tid=2, entry_tid=39392' 2026-02-19T20:31:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=39392' 2026-02-19T20:31:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=39392' 2026-02-19T20:31:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39392 2026-02-19T20:31:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 39392 2026-02-19T20:31:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:08.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=39710], [object_number=37, tag_tid=2, entry_tid=39709], [object_number=36, tag_tid=2, entry_tid=39708], [object_number=39, tag_tid=2, entry_tid=39707]]], state=connected] 2026-02-19T20:31:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=39710], [object_number=37, tag_tid=2, entry_tid=39709], [object_number=36, tag_tid=2, entry_tid=39708], [object_number=39, tag_tid=2, entry_tid=39707]]], state=connected] 2026-02-19T20:31:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=2, entry_tid=39710' 2026-02-19T20:31:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=38, tag_tid=2, entry_tid=39710' 2026-02-19T20:31:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=39710' '!=' 'object_number=36, tag_tid=2, entry_tid=39392' 2026-02-19T20:31:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=39710' = 'object_number=36, tag_tid=2, entry_tid=39392' 2026-02-19T20:31:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=38, tag_tid=2, entry_tid=39710' 2026-02-19T20:31:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=39710' 2026-02-19T20:31:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:08.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=39710' 2026-02-19T20:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39710 2026-02-19T20:31:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 39710 2026-02-19T20:31:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:08.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=40021], [object_number=36, tag_tid=2, entry_tid=40020], [object_number=39, tag_tid=2, entry_tid=40019], [object_number=38, tag_tid=2, entry_tid=40018]]], state=connected] 2026-02-19T20:31:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:08.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:08.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=40021], [object_number=36, tag_tid=2, entry_tid=40020], [object_number=39, tag_tid=2, entry_tid=40019], [object_number=38, tag_tid=2, entry_tid=40018]]], state=connected] 2026-02-19T20:31:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=40021' 2026-02-19T20:31:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=40021' 2026-02-19T20:31:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=40021' '!=' 'object_number=38, tag_tid=2, entry_tid=39710' 2026-02-19T20:31:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=40021' = 'object_number=38, tag_tid=2, entry_tid=39710' 2026-02-19T20:31:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=37, tag_tid=2, entry_tid=40021' 2026-02-19T20:31:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=40021' 2026-02-19T20:31:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=40021' 2026-02-19T20:31:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40021 2026-02-19T20:31:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 40021 2026-02-19T20:31:08.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:08.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:08.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:08.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:08.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:08.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:08.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:08.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:08.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:08.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:08.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=40327], [object_number=38, tag_tid=2, entry_tid=40326], [object_number=37, tag_tid=2, entry_tid=40325], [object_number=36, tag_tid=2, entry_tid=40324]]], state=connected] 2026-02-19T20:31:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=40327], [object_number=38, tag_tid=2, entry_tid=40326], [object_number=37, tag_tid=2, entry_tid=40325], [object_number=36, tag_tid=2, entry_tid=40324]]], state=connected] 2026-02-19T20:31:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:08.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=2, entry_tid=40327' 2026-02-19T20:31:08.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=39, tag_tid=2, entry_tid=40327' 2026-02-19T20:31:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=40327' '!=' 'object_number=37, tag_tid=2, entry_tid=40021' 2026-02-19T20:31:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=40327' = 'object_number=37, tag_tid=2, entry_tid=40021' 2026-02-19T20:31:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=39, tag_tid=2, entry_tid=40327' 2026-02-19T20:31:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:08.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=40327' 2026-02-19T20:31:08.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:08.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=40327' 2026-02-19T20:31:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:08.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40327 2026-02-19T20:31:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 40327 2026-02-19T20:31:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:09.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:09.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:09.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:09.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:09.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:09.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:09.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=40646], [object_number=37, tag_tid=2, entry_tid=40645], [object_number=36, tag_tid=2, entry_tid=40644], [object_number=39, tag_tid=2, entry_tid=40643]]], state=connected] 2026-02-19T20:31:09.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:09.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:09.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:09.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:09.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=40646], [object_number=37, tag_tid=2, entry_tid=40645], [object_number=36, tag_tid=2, entry_tid=40644], [object_number=39, tag_tid=2, entry_tid=40643]]], state=connected] 2026-02-19T20:31:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=2, entry_tid=40646' 2026-02-19T20:31:09.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=38, tag_tid=2, entry_tid=40646' 2026-02-19T20:31:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=40646' '!=' 'object_number=39, tag_tid=2, entry_tid=40327' 2026-02-19T20:31:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=40646' = 'object_number=39, tag_tid=2, entry_tid=40327' 2026-02-19T20:31:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=38, tag_tid=2, entry_tid=40646' 2026-02-19T20:31:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=40646' 2026-02-19T20:31:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:09.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=40646' 2026-02-19T20:31:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:09.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40646 2026-02-19T20:31:09.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 40646 2026-02-19T20:31:09.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:09.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:09.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:09.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:09.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:09.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:09.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:09.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:09.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:09.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:09.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=40975], [object_number=38, tag_tid=2, entry_tid=40974], [object_number=37, tag_tid=2, entry_tid=40973], [object_number=36, tag_tid=2, entry_tid=40972]]], state=connected] 2026-02-19T20:31:09.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=40975], [object_number=38, tag_tid=2, entry_tid=40974], [object_number=37, tag_tid=2, entry_tid=40973], [object_number=36, tag_tid=2, entry_tid=40972]]], state=connected] 2026-02-19T20:31:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=2, entry_tid=40975' 2026-02-19T20:31:09.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=39, tag_tid=2, entry_tid=40975' 2026-02-19T20:31:09.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=40975' '!=' 'object_number=38, tag_tid=2, entry_tid=40646' 2026-02-19T20:31:09.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:09.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=40975' = 'object_number=38, tag_tid=2, entry_tid=40646' 2026-02-19T20:31:09.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=39, tag_tid=2, entry_tid=40975' 2026-02-19T20:31:09.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:09.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:09.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:09.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=40975' 2026-02-19T20:31:09.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:09.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:09.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:09.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:09.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:09.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=40975' 2026-02-19T20:31:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40975 2026-02-19T20:31:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 40975 2026-02-19T20:31:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:09.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:09.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:09.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:09.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:09.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:09.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=41068], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066], [object_number=37, tag_tid=2, entry_tid=41065]]], state=connected] 2026-02-19T20:31:09.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:09.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:09.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:09.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:09.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:09.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:09.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:09.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:09.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:09.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:09.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=41068], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066], [object_number=37, tag_tid=2, entry_tid=41065]]], state=connected] 2026-02-19T20:31:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:10.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=36, tag_tid=2, entry_tid=41068' 2026-02-19T20:31:10.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=36, tag_tid=2, entry_tid=41068' 2026-02-19T20:31:10.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=41068' '!=' 'object_number=39, tag_tid=2, entry_tid=40975' 2026-02-19T20:31:10.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:10.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=41068' = 'object_number=39, tag_tid=2, entry_tid=40975' 2026-02-19T20:31:10.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=36, tag_tid=2, entry_tid=41068' 2026-02-19T20:31:10.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:10.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:10.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:10.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:31:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=41068' 2026-02-19T20:31:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:31:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=41068' 2026-02-19T20:31:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41068 2026-02-19T20:31:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 41068 2026-02-19T20:31:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:10.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:10.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:10.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:10.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:10.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:10.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=41068], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066], [object_number=37, tag_tid=2, entry_tid=41065]]], state=connected] 2026-02-19T20:31:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:10.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:10.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=41068], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066], [object_number=37, tag_tid=2, entry_tid=41065]]], state=connected] 2026-02-19T20:31:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=36, tag_tid=2, entry_tid=41068' 2026-02-19T20:31:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=36, tag_tid=2, entry_tid=41068' 2026-02-19T20:31:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=41068' '!=' 'object_number=36, tag_tid=2, entry_tid=41068' 2026-02-19T20:31:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:31:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:11.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:11.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:11.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:11.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:11.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:11.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:11.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:11.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:11.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:11.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:31:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:31:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=3, entry_tid=1], [object_number=36, tag_tid=3, entry_tid=0], [object_number=39, tag_tid=2, entry_tid=41067], [object_number=38, tag_tid=2, entry_tid=41066]]], state=connected] 2026-02-19T20:31:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=3, entry_tid=1' -a 'object_number=37, tag_tid=3, entry_tid=1' = 'object_number=37, tag_tid=3, entry_tid=1' 2026-02-19T20:31:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2026-02-19T20:31:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap1 2026-02-19T20:31:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:31:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:31:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:31:12.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2026-02-19T20:31:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:31:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:31:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:31:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:31:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:31:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:31:12.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:12.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:31:12.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap1 2026-02-19T20:31:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap1 2026-02-19T20:31:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:31:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:31:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:31:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:31:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:31:12.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:31:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:31:13.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:31:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:31:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:31:13.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name=snap1-clean 2026-02-19T20:31:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap1-clean 2026-02-19T20:31:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:31:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap1-clean 2026-02-19T20:31:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2026-02-19T20:31:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:31:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror test 2026-02-19T20:31:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:31:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:31:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=9 2026-02-19T20:31:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:31:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-19T20:31:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 14856192 bytes 2026-02-19T20:31:14.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 61705216 bytes 2026-02-19T20:31:15.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 114693120 bytes 2026-02-19T20:31:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 181058560 bytes 2026-02-19T20:31:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 227751936 bytes 2026-02-19T20:31:16.583 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T20:31:16.583 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T20:31:16.583 INFO:tasks.rbd_mirror_thrash:waiting for 8.32472123570284 secs before reviving daemons 2026-02-19T20:31:16.584 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T20:31:16.578+0000 7fc67d123640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 19424) UID: 1000 2026-02-19T20:31:16.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 271649792 bytes 2026-02-19T20:31:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 272241664 bytes 2026-02-19T20:31:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 286043136 bytes 2026-02-19T20:31:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 364523520 bytes 2026-02-19T20:31:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 407877632 bytes 2026-02-19T20:31:18.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 430887936 bytes 2026-02-19T20:31:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 488797184 bytes 2026-02-19T20:31:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 609807360 bytes 2026-02-19T20:31:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 645183488 bytes 2026-02-19T20:31:21.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 676534272 bytes 2026-02-19T20:31:21.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 694457344 bytes 2026-02-19T20:31:21.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 708132864 bytes 2026-02-19T20:31:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 756580352 bytes 2026-02-19T20:31:22.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:31:22.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:31:22.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:31:22.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:22.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-19T20:31:22.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:22.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:22.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:22.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:31:22.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:31:22.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:31:22.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:22.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:31:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-19T20:31:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:31:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:31:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:31:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:31:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:31:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-19T20:31:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='test: 2026-02-19T20:31:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: global_id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:31:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+replaying 2026-02-19T20:31:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":27679,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":37,"tag_tid":4},"primary_position":{"entry_tid":27679,"object_number":67,"tag_tid":5}} 2026-02-19T20:31:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: service: mirror.1 on trial151 2026-02-19T20:31:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:31:20 2026-02-19T20:31:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: peer_sites: 2026-02-19T20:31:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: name: cluster2 2026-02-19T20:31:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+stopped 2026-02-19T20:31:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: local image is primary 2026-02-19T20:31:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:31:17' 2026-02-19T20:31:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ state: .*up+replaying' 2026-02-19T20:31:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: state: up+replaying 2026-02-19T20:31:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ description: .*' 2026-02-19T20:31:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":27679,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":37,"tag_tid":4},"primary_position":{"entry_tid":27679,"object_number":67,"tag_tid":5}} 2026-02-19T20:31:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:31:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:31:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^up+' 2026-02-19T20:31:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:31:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep 'service: *mirror..* on ' 2026-02-19T20:31:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: service: mirror.1 on trial151 2026-02-19T20:31:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update 2026-02-19T20:31:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-19T20:31:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:31:20' 2026-02-19T20:31:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-19 20:31:20' 2026-02-19T20:31:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:31:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'prev_last_update=2026-02-19 20:31:20' 2026-02-19T20:31:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:31:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:31:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-19T20:31:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='OKOKOK142064262mirror.1trial15120.3.0-5336-g802bf111falseOK42104271mirror.2trial15120.3.0-5336-g802bf111trueOK42144267mirror.3trial15120.3.0-5336-g802bf111falseOKtest959deb13-de1a-48d9-a341-1f95416f8379up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":27679,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":37,"tag_tid":4},"primary_position":{"entry_tid":27679,"object_number":67,"tag_tid":5}}42064262mirror.1trial1512026-02-19 20:31:20cluster2a054b997-3cfb-48e6-978a-70d6a85e56deup+stoppedlocal image is primary2026-02-19 20:31:17' 2026-02-19T20:31:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update state 2026-02-19T20:31:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-19T20:31:23.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:31:20' 2026-02-19T20:31:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-19T20:31:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=up+replaying 2026-02-19T20:31:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:31:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep up+replaying 2026-02-19T20:31:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:31:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap2 2026-02-19T20:31:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap2 2026-02-19T20:31:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:31:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap2 2026-02-19T20:31:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:31:24.909 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T20:31:24.909 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:31:24.909 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T20:31:24.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T20:31:24.909 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T20:31:24.911 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T20:31:24.911 INFO:tasks.rbd_mirror_thrash:waiting for 116.27189009740178 secs before thrashing 2026-02-19T20:31:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:31:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:31:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:31:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:31:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:55.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:31:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:31:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:31:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:31:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:31:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:31:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:31:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:31:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:31:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:31:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:31:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:31:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:31:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:31:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:31:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:31:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:31:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:31:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:31:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:31:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:31:56.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:31:56.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:31:56.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:31:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:31:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:31:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:31:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:31:56.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:31:56.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:31:56.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:56.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:56.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:56.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:56.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:31:56.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:31:56.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:31:56.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:31:56.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:31:56.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:31:56.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:31:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:31:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:31:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:31:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:31:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:31:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:31:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:31:56.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:56.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:31:56.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:31:56.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:31:56.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:31:56.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:31:56.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:56.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:56.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:56.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:31:56.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:31:56.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:31:56.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:31:56.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:31:56.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:31:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:31:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:31:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:31:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:31:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:31:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:31:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:31:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:56.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:57.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:57.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:57.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:57.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:31:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:31:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:31:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:31:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:31:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:31:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:31:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:31:57.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:57.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:57.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:57.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:57.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:57.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:57.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:57.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:57.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:57.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:57.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:57.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:57.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:57.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:57.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 16 2026-02-19T20:31:57.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:57.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:57.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:57.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=65, tag_tid=5, entry_tid=28605], [object_number=64, tag_tid=5, entry_tid=28604], [object_number=67, tag_tid=5, entry_tid=28603], [object_number=66, tag_tid=5, entry_tid=28602]]], state=connected] 2026-02-19T20:31:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:57.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:57.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:57.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:57.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:57.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 16 2026-02-19T20:31:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=65, tag_tid=5, entry_tid=28605], [object_number=64, tag_tid=5, entry_tid=28604], [object_number=67, tag_tid=5, entry_tid=28603], [object_number=66, tag_tid=5, entry_tid=28602]]], state=connected] 2026-02-19T20:31:57.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=65, tag_tid=5, entry_tid=28605' 2026-02-19T20:31:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=65, tag_tid=5, entry_tid=28605' 2026-02-19T20:31:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=65, tag_tid=5, entry_tid=28605' '!=' '' 2026-02-19T20:31:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=65, tag_tid=5, entry_tid=28605' = '' 2026-02-19T20:31:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=65, tag_tid=5, entry_tid=28605' 2026-02-19T20:31:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:31:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=65, tag_tid=5, entry_tid=28605' 2026-02-19T20:31:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:31:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:57.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:57.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=65, tag_tid=5, entry_tid=28605' 2026-02-19T20:31:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28605 2026-02-19T20:31:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 28605 2026-02-19T20:31:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:57.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:31:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:31:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:31:57.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:31:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:31:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:31:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:31:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:57.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:31:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:31:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:31:57.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:57.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:57.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:57.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:31:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 16 2026-02-19T20:31:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=64, tag_tid=5, entry_tid=28960], [object_number=67, tag_tid=5, entry_tid=28959], [object_number=66, tag_tid=5, entry_tid=28958], [object_number=65, tag_tid=5, entry_tid=28957]]], state=connected] 2026-02-19T20:31:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:57.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:57.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:57.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 16 2026-02-19T20:31:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=64, tag_tid=5, entry_tid=28960], [object_number=67, tag_tid=5, entry_tid=28959], [object_number=66, tag_tid=5, entry_tid=28958], [object_number=65, tag_tid=5, entry_tid=28957]]], state=connected] 2026-02-19T20:31:57.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=64, tag_tid=5, entry_tid=28960' 2026-02-19T20:31:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=64, tag_tid=5, entry_tid=28960' 2026-02-19T20:31:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=64, tag_tid=5, entry_tid=28960' '!=' 'object_number=65, tag_tid=5, entry_tid=28605' 2026-02-19T20:31:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=64, tag_tid=5, entry_tid=28960' = 'object_number=65, tag_tid=5, entry_tid=28605' 2026-02-19T20:31:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=64, tag_tid=5, entry_tid=28960' 2026-02-19T20:31:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:57.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:31:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=64, tag_tid=5, entry_tid=28960' 2026-02-19T20:31:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:57.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:57.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:31:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:57.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:57.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=64, tag_tid=5, entry_tid=28960' 2026-02-19T20:31:57.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:57.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28960 2026-02-19T20:31:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 28960 2026-02-19T20:31:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:57.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:57.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:57.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:57.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:57.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:57.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:31:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:31:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:31:58.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:31:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:58.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:31:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:31:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:31:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:31:58.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:58.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:58.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 16 2026-02-19T20:31:58.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:58.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:58.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:58.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=65, tag_tid=5, entry_tid=29273], [object_number=64, tag_tid=5, entry_tid=29272], [object_number=67, tag_tid=5, entry_tid=29271], [object_number=66, tag_tid=5, entry_tid=29270]]], state=connected] 2026-02-19T20:31:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:58.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:58.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:58.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:58.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:58.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 16 2026-02-19T20:31:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:58.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:58.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=65, tag_tid=5, entry_tid=29273], [object_number=64, tag_tid=5, entry_tid=29272], [object_number=67, tag_tid=5, entry_tid=29271], [object_number=66, tag_tid=5, entry_tid=29270]]], state=connected] 2026-02-19T20:31:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=65, tag_tid=5, entry_tid=29273' 2026-02-19T20:31:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=65, tag_tid=5, entry_tid=29273' 2026-02-19T20:31:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=65, tag_tid=5, entry_tid=29273' '!=' 'object_number=64, tag_tid=5, entry_tid=28960' 2026-02-19T20:31:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:58.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=65, tag_tid=5, entry_tid=29273' = 'object_number=64, tag_tid=5, entry_tid=28960' 2026-02-19T20:31:58.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=65, tag_tid=5, entry_tid=29273' 2026-02-19T20:31:58.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:58.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:31:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=65, tag_tid=5, entry_tid=29273' 2026-02-19T20:31:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:31:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:58.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:58.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=65, tag_tid=5, entry_tid=29273' 2026-02-19T20:31:58.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:58.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:58.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29273 2026-02-19T20:31:58.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 29273 2026-02-19T20:31:58.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:58.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:58.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:58.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:58.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:58.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:58.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:58.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:58.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:58.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:58.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:58.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:58.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:58.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:58.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:58.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:58.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:58.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:58.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:31:58.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:31:58.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:31:58.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:58.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:31:58.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:31:58.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:58.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:58.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:58.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:31:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:31:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:31:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:31:58.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 16 2026-02-19T20:31:58.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:58.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=64, tag_tid=5, entry_tid=29600], [object_number=67, tag_tid=5, entry_tid=29599], [object_number=66, tag_tid=5, entry_tid=29598], [object_number=65, tag_tid=5, entry_tid=29597]]], state=connected] 2026-02-19T20:31:58.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 16 2026-02-19T20:31:58.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:58.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:58.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:58.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=64, tag_tid=5, entry_tid=29600], [object_number=67, tag_tid=5, entry_tid=29599], [object_number=66, tag_tid=5, entry_tid=29598], [object_number=65, tag_tid=5, entry_tid=29597]]], state=connected] 2026-02-19T20:31:58.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=64, tag_tid=5, entry_tid=29600' 2026-02-19T20:31:58.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=64, tag_tid=5, entry_tid=29600' 2026-02-19T20:31:58.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=64, tag_tid=5, entry_tid=29600' '!=' 'object_number=65, tag_tid=5, entry_tid=29273' 2026-02-19T20:31:58.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:58.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=64, tag_tid=5, entry_tid=29600' = 'object_number=65, tag_tid=5, entry_tid=29273' 2026-02-19T20:31:58.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=64, tag_tid=5, entry_tid=29600' 2026-02-19T20:31:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:58.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:31:58.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=64, tag_tid=5, entry_tid=29600' 2026-02-19T20:31:58.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:58.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:58.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:31:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:58.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:58.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=64, tag_tid=5, entry_tid=29600' 2026-02-19T20:31:58.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:58.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29600 2026-02-19T20:31:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 29600 2026-02-19T20:31:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:58.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:58.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:58.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:58.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:58.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:58.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:58.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:31:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:31:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:31:58.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:58.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:31:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:31:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:31:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:31:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:31:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:58.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:58.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:58.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:58.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:58.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:58.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 16 2026-02-19T20:31:58.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:58.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:58.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:58.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=66, tag_tid=5, entry_tid=29918], [object_number=65, tag_tid=5, entry_tid=29917], [object_number=64, tag_tid=5, entry_tid=29916], [object_number=67, tag_tid=5, entry_tid=29915]]], state=connected] 2026-02-19T20:31:58.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:58.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 16 2026-02-19T20:31:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=66, tag_tid=5, entry_tid=29918], [object_number=65, tag_tid=5, entry_tid=29917], [object_number=64, tag_tid=5, entry_tid=29916], [object_number=67, tag_tid=5, entry_tid=29915]]], state=connected] 2026-02-19T20:31:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:58.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=66, tag_tid=5, entry_tid=29918' 2026-02-19T20:31:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=66, tag_tid=5, entry_tid=29918' 2026-02-19T20:31:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=66, tag_tid=5, entry_tid=29918' '!=' 'object_number=64, tag_tid=5, entry_tid=29600' 2026-02-19T20:31:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=66, tag_tid=5, entry_tid=29918' = 'object_number=64, tag_tid=5, entry_tid=29600' 2026-02-19T20:31:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=66, tag_tid=5, entry_tid=29918' 2026-02-19T20:31:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:31:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=66, tag_tid=5, entry_tid=29918' 2026-02-19T20:31:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:31:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=66, tag_tid=5, entry_tid=29918' 2026-02-19T20:31:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:59.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29918 2026-02-19T20:31:59.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 29918 2026-02-19T20:31:59.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:59.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:31:59.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:31:59.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:31:59.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:59.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:31:59.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:31:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:31:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:31:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:31:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:59.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:59.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:31:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=69, tag_tid=5, entry_tid=30133], [object_number=68, tag_tid=5, entry_tid=30132], [object_number=71, tag_tid=5, entry_tid=30131], [object_number=70, tag_tid=5, entry_tid=30130]]], state=connected] 2026-02-19T20:31:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:31:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=69, tag_tid=5, entry_tid=30133], [object_number=68, tag_tid=5, entry_tid=30132], [object_number=71, tag_tid=5, entry_tid=30131], [object_number=70, tag_tid=5, entry_tid=30130]]], state=connected] 2026-02-19T20:31:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=69, tag_tid=5, entry_tid=30133' 2026-02-19T20:31:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=69, tag_tid=5, entry_tid=30133' 2026-02-19T20:31:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=69, tag_tid=5, entry_tid=30133' '!=' 'object_number=66, tag_tid=5, entry_tid=29918' 2026-02-19T20:31:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=69, tag_tid=5, entry_tid=30133' = 'object_number=66, tag_tid=5, entry_tid=29918' 2026-02-19T20:31:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=69, tag_tid=5, entry_tid=30133' 2026-02-19T20:31:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:31:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=69, tag_tid=5, entry_tid=30133' 2026-02-19T20:31:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:59.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:31:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=69, tag_tid=5, entry_tid=30133' 2026-02-19T20:31:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30133 2026-02-19T20:31:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 30133 2026-02-19T20:31:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:31:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:31:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:31:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:31:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:31:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:31:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:31:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:31:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:31:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:31:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:31:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:31:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:31:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:31:59.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:31:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:31:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:31:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:31:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:31:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:31:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:31:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:31:59.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:59.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:59.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:31:59.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:31:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:31:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:31:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:31:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:31:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:31:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:31:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:31:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:31:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:31:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:31:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:31:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:31:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:31:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:31:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:31:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:31:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:59.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:31:59.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:59.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:59.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:59.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:59.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:31:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=70, tag_tid=5, entry_tid=30434], [object_number=69, tag_tid=5, entry_tid=30433], [object_number=68, tag_tid=5, entry_tid=30432], [object_number=71, tag_tid=5, entry_tid=30431]]], state=connected] 2026-02-19T20:31:59.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:31:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:31:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:31:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:31:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:31:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:31:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:31:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:31:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:31:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:31:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:31:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:31:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:31:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:31:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=70, tag_tid=5, entry_tid=30434], [object_number=69, tag_tid=5, entry_tid=30433], [object_number=68, tag_tid=5, entry_tid=30432], [object_number=71, tag_tid=5, entry_tid=30431]]], state=connected] 2026-02-19T20:31:59.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:31:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=70, tag_tid=5, entry_tid=30434' 2026-02-19T20:31:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=70, tag_tid=5, entry_tid=30434' 2026-02-19T20:31:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=70, tag_tid=5, entry_tid=30434' '!=' 'object_number=69, tag_tid=5, entry_tid=30133' 2026-02-19T20:31:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:31:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=70, tag_tid=5, entry_tid=30434' = 'object_number=69, tag_tid=5, entry_tid=30133' 2026-02-19T20:31:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=70, tag_tid=5, entry_tid=30434' 2026-02-19T20:31:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:59.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:31:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=70, tag_tid=5, entry_tid=30434' 2026-02-19T20:31:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:31:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:31:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:31:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:31:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=70, tag_tid=5, entry_tid=30434' 2026-02-19T20:31:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:31:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:31:59.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30434 2026-02-19T20:31:59.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 30434 2026-02-19T20:31:59.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:31:59.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:31:59.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:00.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:00.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=70, tag_tid=5, entry_tid=30742], [object_number=69, tag_tid=5, entry_tid=30741], [object_number=68, tag_tid=5, entry_tid=30740], [object_number=71, tag_tid=5, entry_tid=30739]]], state=connected] 2026-02-19T20:32:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=70, tag_tid=5, entry_tid=30742], [object_number=69, tag_tid=5, entry_tid=30741], [object_number=68, tag_tid=5, entry_tid=30740], [object_number=71, tag_tid=5, entry_tid=30739]]], state=connected] 2026-02-19T20:32:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=70, tag_tid=5, entry_tid=30742' 2026-02-19T20:32:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=70, tag_tid=5, entry_tid=30742' 2026-02-19T20:32:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=70, tag_tid=5, entry_tid=30742' '!=' 'object_number=70, tag_tid=5, entry_tid=30434' 2026-02-19T20:32:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=70, tag_tid=5, entry_tid=30742' = 'object_number=70, tag_tid=5, entry_tid=30434' 2026-02-19T20:32:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=70, tag_tid=5, entry_tid=30742' 2026-02-19T20:32:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=70, tag_tid=5, entry_tid=30742' 2026-02-19T20:32:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:00.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=70, tag_tid=5, entry_tid=30742' 2026-02-19T20:32:00.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:00.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30742 2026-02-19T20:32:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 30742 2026-02-19T20:32:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:00.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:00.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:00.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:00.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:00.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:00.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=69, tag_tid=5, entry_tid=31029], [object_number=68, tag_tid=5, entry_tid=31028], [object_number=71, tag_tid=5, entry_tid=31027], [object_number=70, tag_tid=5, entry_tid=31026]]], state=connected] 2026-02-19T20:32:00.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=69, tag_tid=5, entry_tid=31029], [object_number=68, tag_tid=5, entry_tid=31028], [object_number=71, tag_tid=5, entry_tid=31027], [object_number=70, tag_tid=5, entry_tid=31026]]], state=connected] 2026-02-19T20:32:00.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:00.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=69, tag_tid=5, entry_tid=31029' 2026-02-19T20:32:00.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=69, tag_tid=5, entry_tid=31029' 2026-02-19T20:32:00.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=69, tag_tid=5, entry_tid=31029' '!=' 'object_number=70, tag_tid=5, entry_tid=30742' 2026-02-19T20:32:00.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:00.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=69, tag_tid=5, entry_tid=31029' = 'object_number=70, tag_tid=5, entry_tid=30742' 2026-02-19T20:32:00.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=69, tag_tid=5, entry_tid=31029' 2026-02-19T20:32:00.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:00.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:00.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=69, tag_tid=5, entry_tid=31029' 2026-02-19T20:32:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:00.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:00.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:00.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:00.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:00.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=69, tag_tid=5, entry_tid=31029' 2026-02-19T20:32:00.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:00.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:00.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31029 2026-02-19T20:32:00.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 31029 2026-02-19T20:32:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:01.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=68, tag_tid=5, entry_tid=31376], [object_number=71, tag_tid=5, entry_tid=31375], [object_number=70, tag_tid=5, entry_tid=31374], [object_number=69, tag_tid=5, entry_tid=31373]]], state=connected] 2026-02-19T20:32:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:01.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:01.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:01.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:01.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:01.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=68, tag_tid=5, entry_tid=31376], [object_number=71, tag_tid=5, entry_tid=31375], [object_number=70, tag_tid=5, entry_tid=31374], [object_number=69, tag_tid=5, entry_tid=31373]]], state=connected] 2026-02-19T20:32:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=68, tag_tid=5, entry_tid=31376' 2026-02-19T20:32:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=68, tag_tid=5, entry_tid=31376' 2026-02-19T20:32:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=68, tag_tid=5, entry_tid=31376' '!=' 'object_number=69, tag_tid=5, entry_tid=31029' 2026-02-19T20:32:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=68, tag_tid=5, entry_tid=31376' = 'object_number=69, tag_tid=5, entry_tid=31029' 2026-02-19T20:32:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=68, tag_tid=5, entry_tid=31376' 2026-02-19T20:32:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=68, tag_tid=5, entry_tid=31376' 2026-02-19T20:32:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:01.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=68, tag_tid=5, entry_tid=31376' 2026-02-19T20:32:01.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:01.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31376 2026-02-19T20:32:01.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 31376 2026-02-19T20:32:01.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:01.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:01.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:01.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:01.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:01.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:01.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:01.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:01.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:01.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:01.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:01.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=71, tag_tid=5, entry_tid=31731], [object_number=70, tag_tid=5, entry_tid=31730], [object_number=69, tag_tid=5, entry_tid=31729], [object_number=68, tag_tid=5, entry_tid=31728]]], state=connected] 2026-02-19T20:32:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:01.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:01.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:01.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:01.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:01.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:01.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=71, tag_tid=5, entry_tid=31731], [object_number=70, tag_tid=5, entry_tid=31730], [object_number=69, tag_tid=5, entry_tid=31729], [object_number=68, tag_tid=5, entry_tid=31728]]], state=connected] 2026-02-19T20:32:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=71, tag_tid=5, entry_tid=31731' 2026-02-19T20:32:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=71, tag_tid=5, entry_tid=31731' 2026-02-19T20:32:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=71, tag_tid=5, entry_tid=31731' '!=' 'object_number=68, tag_tid=5, entry_tid=31376' 2026-02-19T20:32:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=71, tag_tid=5, entry_tid=31731' = 'object_number=68, tag_tid=5, entry_tid=31376' 2026-02-19T20:32:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=71, tag_tid=5, entry_tid=31731' 2026-02-19T20:32:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:01.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=71, tag_tid=5, entry_tid=31731' 2026-02-19T20:32:01.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:01.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:01.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:01.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:01.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=71, tag_tid=5, entry_tid=31731' 2026-02-19T20:32:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:01.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31731 2026-02-19T20:32:01.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 31731 2026-02-19T20:32:01.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:01.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:01.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:01.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:01.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:01.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:01.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:01.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:01.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:01.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:01.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:01.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:01.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:01.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:01.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:01.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:01.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:01.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:01.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:01.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:01.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:01.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:01.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=68, tag_tid=5, entry_tid=32068], [object_number=71, tag_tid=5, entry_tid=32067], [object_number=70, tag_tid=5, entry_tid=32066], [object_number=69, tag_tid=5, entry_tid=32065]]], state=connected] 2026-02-19T20:32:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=68, tag_tid=5, entry_tid=32068], [object_number=71, tag_tid=5, entry_tid=32067], [object_number=70, tag_tid=5, entry_tid=32066], [object_number=69, tag_tid=5, entry_tid=32065]]], state=connected] 2026-02-19T20:32:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=68, tag_tid=5, entry_tid=32068' 2026-02-19T20:32:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=68, tag_tid=5, entry_tid=32068' 2026-02-19T20:32:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=68, tag_tid=5, entry_tid=32068' '!=' 'object_number=71, tag_tid=5, entry_tid=31731' 2026-02-19T20:32:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=68, tag_tid=5, entry_tid=32068' = 'object_number=71, tag_tid=5, entry_tid=31731' 2026-02-19T20:32:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=68, tag_tid=5, entry_tid=32068' 2026-02-19T20:32:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=68, tag_tid=5, entry_tid=32068' 2026-02-19T20:32:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:01.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:01.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=68, tag_tid=5, entry_tid=32068' 2026-02-19T20:32:01.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:01.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32068 2026-02-19T20:32:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 32068 2026-02-19T20:32:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:02.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:02.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:02.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:02.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:02.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=71, tag_tid=5, entry_tid=32367], [object_number=70, tag_tid=5, entry_tid=32366], [object_number=69, tag_tid=5, entry_tid=32365], [object_number=68, tag_tid=5, entry_tid=32364]]], state=connected] 2026-02-19T20:32:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:02.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:02.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:02.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=71, tag_tid=5, entry_tid=32367], [object_number=70, tag_tid=5, entry_tid=32366], [object_number=69, tag_tid=5, entry_tid=32365], [object_number=68, tag_tid=5, entry_tid=32364]]], state=connected] 2026-02-19T20:32:02.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=71, tag_tid=5, entry_tid=32367' 2026-02-19T20:32:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=71, tag_tid=5, entry_tid=32367' 2026-02-19T20:32:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=71, tag_tid=5, entry_tid=32367' '!=' 'object_number=68, tag_tid=5, entry_tid=32068' 2026-02-19T20:32:02.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:02.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=71, tag_tid=5, entry_tid=32367' = 'object_number=68, tag_tid=5, entry_tid=32068' 2026-02-19T20:32:02.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=71, tag_tid=5, entry_tid=32367' 2026-02-19T20:32:02.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:02.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:02.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:02.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:02.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=71, tag_tid=5, entry_tid=32367' 2026-02-19T20:32:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:02.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:02.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=71, tag_tid=5, entry_tid=32367' 2026-02-19T20:32:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32367 2026-02-19T20:32:02.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 32367 2026-02-19T20:32:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:02.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:02.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:02.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:02.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:02.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:02.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:02.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:02.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:02.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:02.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:02.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:02.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:02.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:02.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:02.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:02.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:02.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:02.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:02.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:02.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:02.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:02.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:02.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:02.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:02.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:02.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:02.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:02.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:02.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=70, tag_tid=5, entry_tid=32690], [object_number=69, tag_tid=5, entry_tid=32689], [object_number=68, tag_tid=5, entry_tid=32688], [object_number=71, tag_tid=5, entry_tid=32687]]], state=connected] 2026-02-19T20:32:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=70, tag_tid=5, entry_tid=32690], [object_number=69, tag_tid=5, entry_tid=32689], [object_number=68, tag_tid=5, entry_tid=32688], [object_number=71, tag_tid=5, entry_tid=32687]]], state=connected] 2026-02-19T20:32:02.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=70, tag_tid=5, entry_tid=32690' 2026-02-19T20:32:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=70, tag_tid=5, entry_tid=32690' 2026-02-19T20:32:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=70, tag_tid=5, entry_tid=32690' '!=' 'object_number=71, tag_tid=5, entry_tid=32367' 2026-02-19T20:32:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=70, tag_tid=5, entry_tid=32690' = 'object_number=71, tag_tid=5, entry_tid=32367' 2026-02-19T20:32:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=70, tag_tid=5, entry_tid=32690' 2026-02-19T20:32:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=70, tag_tid=5, entry_tid=32690' 2026-02-19T20:32:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:02.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:02.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:02.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:02.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:02.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=70, tag_tid=5, entry_tid=32690' 2026-02-19T20:32:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32690 2026-02-19T20:32:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 32690 2026-02-19T20:32:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:03.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:03.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:03.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:03.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:03.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:03.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:03.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:03.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:03.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:03.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:03.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:03.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=71, tag_tid=5, entry_tid=33007], [object_number=70, tag_tid=5, entry_tid=33006], [object_number=69, tag_tid=5, entry_tid=33005], [object_number=68, tag_tid=5, entry_tid=33004]]], state=connected] 2026-02-19T20:32:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=71, tag_tid=5, entry_tid=33007], [object_number=70, tag_tid=5, entry_tid=33006], [object_number=69, tag_tid=5, entry_tid=33005], [object_number=68, tag_tid=5, entry_tid=33004]]], state=connected] 2026-02-19T20:32:03.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=71, tag_tid=5, entry_tid=33007' 2026-02-19T20:32:03.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=71, tag_tid=5, entry_tid=33007' 2026-02-19T20:32:03.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=71, tag_tid=5, entry_tid=33007' '!=' 'object_number=70, tag_tid=5, entry_tid=32690' 2026-02-19T20:32:03.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:03.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=71, tag_tid=5, entry_tid=33007' = 'object_number=70, tag_tid=5, entry_tid=32690' 2026-02-19T20:32:03.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=71, tag_tid=5, entry_tid=33007' 2026-02-19T20:32:03.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:03.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:03.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:03.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:03.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=71, tag_tid=5, entry_tid=33007' 2026-02-19T20:32:03.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:03.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:03.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:03.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=71, tag_tid=5, entry_tid=33007' 2026-02-19T20:32:03.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:03.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33007 2026-02-19T20:32:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 33007 2026-02-19T20:32:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:03.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:03.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:03.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:03.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:03.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:03.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:03.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:03.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:03.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=71, tag_tid=5, entry_tid=33323], [object_number=70, tag_tid=5, entry_tid=33322], [object_number=69, tag_tid=5, entry_tid=33321], [object_number=68, tag_tid=5, entry_tid=33320]]], state=connected] 2026-02-19T20:32:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:03.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:03.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:03.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:03.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:03.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:03.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:03.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:03.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:03.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:03.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:03.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=71, tag_tid=5, entry_tid=33323], [object_number=70, tag_tid=5, entry_tid=33322], [object_number=69, tag_tid=5, entry_tid=33321], [object_number=68, tag_tid=5, entry_tid=33320]]], state=connected] 2026-02-19T20:32:03.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=71, tag_tid=5, entry_tid=33323' 2026-02-19T20:32:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=71, tag_tid=5, entry_tid=33323' 2026-02-19T20:32:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=71, tag_tid=5, entry_tid=33323' '!=' 'object_number=71, tag_tid=5, entry_tid=33007' 2026-02-19T20:32:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=71, tag_tid=5, entry_tid=33323' = 'object_number=71, tag_tid=5, entry_tid=33007' 2026-02-19T20:32:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=71, tag_tid=5, entry_tid=33323' 2026-02-19T20:32:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:03.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:03.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=71, tag_tid=5, entry_tid=33323' 2026-02-19T20:32:03.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:03.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:03.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:03.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=71, tag_tid=5, entry_tid=33323' 2026-02-19T20:32:03.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:03.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33323 2026-02-19T20:32:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 33323 2026-02-19T20:32:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:03.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:03.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=70, tag_tid=5, entry_tid=33678], [object_number=69, tag_tid=5, entry_tid=33677], [object_number=68, tag_tid=5, entry_tid=33676], [object_number=71, tag_tid=5, entry_tid=33675]]], state=connected] 2026-02-19T20:32:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=70, tag_tid=5, entry_tid=33678], [object_number=69, tag_tid=5, entry_tid=33677], [object_number=68, tag_tid=5, entry_tid=33676], [object_number=71, tag_tid=5, entry_tid=33675]]], state=connected] 2026-02-19T20:32:04.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=70, tag_tid=5, entry_tid=33678' 2026-02-19T20:32:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=70, tag_tid=5, entry_tid=33678' 2026-02-19T20:32:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=70, tag_tid=5, entry_tid=33678' '!=' 'object_number=71, tag_tid=5, entry_tid=33323' 2026-02-19T20:32:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=70, tag_tid=5, entry_tid=33678' = 'object_number=71, tag_tid=5, entry_tid=33323' 2026-02-19T20:32:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=70, tag_tid=5, entry_tid=33678' 2026-02-19T20:32:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=70, tag_tid=5, entry_tid=33678' 2026-02-19T20:32:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:04.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:04.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:04.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:04.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=70, tag_tid=5, entry_tid=33678' 2026-02-19T20:32:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33678 2026-02-19T20:32:04.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 33678 2026-02-19T20:32:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:04.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:04.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:04.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:04.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:04.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=71, tag_tid=5, entry_tid=34039], [object_number=70, tag_tid=5, entry_tid=34038], [object_number=69, tag_tid=5, entry_tid=34037], [object_number=68, tag_tid=5, entry_tid=34036]]], state=connected] 2026-02-19T20:32:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 17 2026-02-19T20:32:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=71, tag_tid=5, entry_tid=34039], [object_number=70, tag_tid=5, entry_tid=34038], [object_number=69, tag_tid=5, entry_tid=34037], [object_number=68, tag_tid=5, entry_tid=34036]]], state=connected] 2026-02-19T20:32:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=71, tag_tid=5, entry_tid=34039' 2026-02-19T20:32:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=71, tag_tid=5, entry_tid=34039' 2026-02-19T20:32:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=71, tag_tid=5, entry_tid=34039' '!=' 'object_number=70, tag_tid=5, entry_tid=33678' 2026-02-19T20:32:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=71, tag_tid=5, entry_tid=34039' = 'object_number=70, tag_tid=5, entry_tid=33678' 2026-02-19T20:32:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=71, tag_tid=5, entry_tid=34039' 2026-02-19T20:32:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=71, tag_tid=5, entry_tid=34039' 2026-02-19T20:32:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:04.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=71, tag_tid=5, entry_tid=34039' 2026-02-19T20:32:04.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:04.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34039 2026-02-19T20:32:04.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 34039 2026-02-19T20:32:04.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:04.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:04.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:04.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:04.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:04.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:04.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:04.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:04.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:04.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:04.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:04.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:04.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=74, tag_tid=5, entry_tid=34282], [object_number=73, tag_tid=5, entry_tid=34281], [object_number=72, tag_tid=5, entry_tid=34280], [object_number=75, tag_tid=5, entry_tid=34279]]], state=connected] 2026-02-19T20:32:04.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:04.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:04.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:04.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:04.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:04.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:04.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:04.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=74, tag_tid=5, entry_tid=34282], [object_number=73, tag_tid=5, entry_tid=34281], [object_number=72, tag_tid=5, entry_tid=34280], [object_number=75, tag_tid=5, entry_tid=34279]]], state=connected] 2026-02-19T20:32:04.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:04.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=74, tag_tid=5, entry_tid=34282' 2026-02-19T20:32:04.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=74, tag_tid=5, entry_tid=34282' 2026-02-19T20:32:04.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=74, tag_tid=5, entry_tid=34282' '!=' 'object_number=71, tag_tid=5, entry_tid=34039' 2026-02-19T20:32:04.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:04.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=74, tag_tid=5, entry_tid=34282' = 'object_number=71, tag_tid=5, entry_tid=34039' 2026-02-19T20:32:04.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=74, tag_tid=5, entry_tid=34282' 2026-02-19T20:32:04.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:04.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:04.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=74, tag_tid=5, entry_tid=34282' 2026-02-19T20:32:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=74, tag_tid=5, entry_tid=34282' 2026-02-19T20:32:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:04.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34282 2026-02-19T20:32:04.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 34282 2026-02-19T20:32:04.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:04.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:04.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:05.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:05.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:05.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:05.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:05.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:05.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:05.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:05.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=73, tag_tid=5, entry_tid=34597], [object_number=72, tag_tid=5, entry_tid=34596], [object_number=75, tag_tid=5, entry_tid=34595], [object_number=74, tag_tid=5, entry_tid=34594]]], state=connected] 2026-02-19T20:32:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=73, tag_tid=5, entry_tid=34597], [object_number=72, tag_tid=5, entry_tid=34596], [object_number=75, tag_tid=5, entry_tid=34595], [object_number=74, tag_tid=5, entry_tid=34594]]], state=connected] 2026-02-19T20:32:05.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=73, tag_tid=5, entry_tid=34597' 2026-02-19T20:32:05.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=73, tag_tid=5, entry_tid=34597' 2026-02-19T20:32:05.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=73, tag_tid=5, entry_tid=34597' '!=' 'object_number=74, tag_tid=5, entry_tid=34282' 2026-02-19T20:32:05.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:05.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=73, tag_tid=5, entry_tid=34597' = 'object_number=74, tag_tid=5, entry_tid=34282' 2026-02-19T20:32:05.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=73, tag_tid=5, entry_tid=34597' 2026-02-19T20:32:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:05.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=73, tag_tid=5, entry_tid=34597' 2026-02-19T20:32:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:05.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:05.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=73, tag_tid=5, entry_tid=34597' 2026-02-19T20:32:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34597 2026-02-19T20:32:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 34597 2026-02-19T20:32:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:05.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:05.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:05.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:05.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:05.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:05.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:05.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:05.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:05.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:05.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:05.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:05.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:05.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:05.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:05.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:05.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:05.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:05.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:05.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:05.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:05.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:05.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:05.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:05.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:05.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=73, tag_tid=5, entry_tid=34921], [object_number=72, tag_tid=5, entry_tid=34920], [object_number=75, tag_tid=5, entry_tid=34919], [object_number=74, tag_tid=5, entry_tid=34918]]], state=connected] 2026-02-19T20:32:05.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:05.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:05.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=73, tag_tid=5, entry_tid=34921], [object_number=72, tag_tid=5, entry_tid=34920], [object_number=75, tag_tid=5, entry_tid=34919], [object_number=74, tag_tid=5, entry_tid=34918]]], state=connected] 2026-02-19T20:32:05.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=73, tag_tid=5, entry_tid=34921' 2026-02-19T20:32:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=73, tag_tid=5, entry_tid=34921' 2026-02-19T20:32:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=73, tag_tid=5, entry_tid=34921' '!=' 'object_number=73, tag_tid=5, entry_tid=34597' 2026-02-19T20:32:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=73, tag_tid=5, entry_tid=34921' = 'object_number=73, tag_tid=5, entry_tid=34597' 2026-02-19T20:32:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=73, tag_tid=5, entry_tid=34921' 2026-02-19T20:32:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:05.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:05.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=73, tag_tid=5, entry_tid=34921' 2026-02-19T20:32:05.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:05.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:05.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=73, tag_tid=5, entry_tid=34921' 2026-02-19T20:32:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34921 2026-02-19T20:32:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 34921 2026-02-19T20:32:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:06.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:06.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:06.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:06.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:06.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:06.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:06.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:06.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:06.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:06.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:06.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:06.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:06.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:06.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:06.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:06.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:06.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:06.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:06.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:06.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:06.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:06.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:06.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=74, tag_tid=5, entry_tid=35290], [object_number=73, tag_tid=5, entry_tid=35289], [object_number=72, tag_tid=5, entry_tid=35288], [object_number=75, tag_tid=5, entry_tid=35287]]], state=connected] 2026-02-19T20:32:06.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=74, tag_tid=5, entry_tid=35290], [object_number=73, tag_tid=5, entry_tid=35289], [object_number=72, tag_tid=5, entry_tid=35288], [object_number=75, tag_tid=5, entry_tid=35287]]], state=connected] 2026-02-19T20:32:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=74, tag_tid=5, entry_tid=35290' 2026-02-19T20:32:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=74, tag_tid=5, entry_tid=35290' 2026-02-19T20:32:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=74, tag_tid=5, entry_tid=35290' '!=' 'object_number=73, tag_tid=5, entry_tid=34921' 2026-02-19T20:32:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=74, tag_tid=5, entry_tid=35290' = 'object_number=73, tag_tid=5, entry_tid=34921' 2026-02-19T20:32:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=74, tag_tid=5, entry_tid=35290' 2026-02-19T20:32:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=74, tag_tid=5, entry_tid=35290' 2026-02-19T20:32:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=74, tag_tid=5, entry_tid=35290' 2026-02-19T20:32:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35290 2026-02-19T20:32:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 35290 2026-02-19T20:32:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:06.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:06.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:06.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:06.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:06.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:06.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:06.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:06.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:06.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:06.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:06.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:06.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:06.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:06.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=73, tag_tid=5, entry_tid=35645], [object_number=72, tag_tid=5, entry_tid=35644], [object_number=75, tag_tid=5, entry_tid=35643], [object_number=74, tag_tid=5, entry_tid=35642]]], state=connected] 2026-02-19T20:32:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:06.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:06.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:06.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:06.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:06.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:06.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:06.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=73, tag_tid=5, entry_tid=35645], [object_number=72, tag_tid=5, entry_tid=35644], [object_number=75, tag_tid=5, entry_tid=35643], [object_number=74, tag_tid=5, entry_tid=35642]]], state=connected] 2026-02-19T20:32:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=73, tag_tid=5, entry_tid=35645' 2026-02-19T20:32:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=73, tag_tid=5, entry_tid=35645' 2026-02-19T20:32:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=73, tag_tid=5, entry_tid=35645' '!=' 'object_number=74, tag_tid=5, entry_tid=35290' 2026-02-19T20:32:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=73, tag_tid=5, entry_tid=35645' = 'object_number=74, tag_tid=5, entry_tid=35290' 2026-02-19T20:32:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=73, tag_tid=5, entry_tid=35645' 2026-02-19T20:32:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=73, tag_tid=5, entry_tid=35645' 2026-02-19T20:32:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=73, tag_tid=5, entry_tid=35645' 2026-02-19T20:32:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:06.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35645 2026-02-19T20:32:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 35645 2026-02-19T20:32:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:06.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:06.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:06.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:06.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:06.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:06.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:06.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:06.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:06.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:06.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:06.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:06.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:06.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:06.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:06.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:06.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:07.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:07.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:07.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:07.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=75, tag_tid=5, entry_tid=35955], [object_number=74, tag_tid=5, entry_tid=35954], [object_number=73, tag_tid=5, entry_tid=35953], [object_number=72, tag_tid=5, entry_tid=35952]]], state=connected] 2026-02-19T20:32:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:07.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:07.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:07.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:07.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:07.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:07.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=75, tag_tid=5, entry_tid=35955], [object_number=74, tag_tid=5, entry_tid=35954], [object_number=73, tag_tid=5, entry_tid=35953], [object_number=72, tag_tid=5, entry_tid=35952]]], state=connected] 2026-02-19T20:32:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=75, tag_tid=5, entry_tid=35955' 2026-02-19T20:32:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=75, tag_tid=5, entry_tid=35955' 2026-02-19T20:32:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=75, tag_tid=5, entry_tid=35955' '!=' 'object_number=73, tag_tid=5, entry_tid=35645' 2026-02-19T20:32:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=75, tag_tid=5, entry_tid=35955' = 'object_number=73, tag_tid=5, entry_tid=35645' 2026-02-19T20:32:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=75, tag_tid=5, entry_tid=35955' 2026-02-19T20:32:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=75, tag_tid=5, entry_tid=35955' 2026-02-19T20:32:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:07.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=75, tag_tid=5, entry_tid=35955' 2026-02-19T20:32:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35955 2026-02-19T20:32:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 35955 2026-02-19T20:32:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:07.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:07.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:07.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:07.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:07.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:07.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:07.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:07.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:07.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:07.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:07.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=73, tag_tid=5, entry_tid=36289], [object_number=72, tag_tid=5, entry_tid=36288], [object_number=75, tag_tid=5, entry_tid=36287], [object_number=74, tag_tid=5, entry_tid=36286]]], state=connected] 2026-02-19T20:32:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:07.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:07.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:07.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=73, tag_tid=5, entry_tid=36289], [object_number=72, tag_tid=5, entry_tid=36288], [object_number=75, tag_tid=5, entry_tid=36287], [object_number=74, tag_tid=5, entry_tid=36286]]], state=connected] 2026-02-19T20:32:07.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=73, tag_tid=5, entry_tid=36289' 2026-02-19T20:32:07.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=73, tag_tid=5, entry_tid=36289' 2026-02-19T20:32:07.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=73, tag_tid=5, entry_tid=36289' '!=' 'object_number=75, tag_tid=5, entry_tid=35955' 2026-02-19T20:32:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=73, tag_tid=5, entry_tid=36289' = 'object_number=75, tag_tid=5, entry_tid=35955' 2026-02-19T20:32:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=73, tag_tid=5, entry_tid=36289' 2026-02-19T20:32:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=73, tag_tid=5, entry_tid=36289' 2026-02-19T20:32:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:07.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:07.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=73, tag_tid=5, entry_tid=36289' 2026-02-19T20:32:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36289 2026-02-19T20:32:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 36289 2026-02-19T20:32:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:07.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:07.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:07.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:07.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:07.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:07.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=74, tag_tid=5, entry_tid=36646], [object_number=73, tag_tid=5, entry_tid=36645], [object_number=72, tag_tid=5, entry_tid=36644], [object_number=75, tag_tid=5, entry_tid=36643]]], state=connected] 2026-02-19T20:32:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:07.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:07.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:07.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:07.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:07.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=74, tag_tid=5, entry_tid=36646], [object_number=73, tag_tid=5, entry_tid=36645], [object_number=72, tag_tid=5, entry_tid=36644], [object_number=75, tag_tid=5, entry_tid=36643]]], state=connected] 2026-02-19T20:32:07.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:07.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=74, tag_tid=5, entry_tid=36646' 2026-02-19T20:32:07.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=74, tag_tid=5, entry_tid=36646' 2026-02-19T20:32:07.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=74, tag_tid=5, entry_tid=36646' '!=' 'object_number=73, tag_tid=5, entry_tid=36289' 2026-02-19T20:32:07.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:07.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=74, tag_tid=5, entry_tid=36646' = 'object_number=73, tag_tid=5, entry_tid=36289' 2026-02-19T20:32:07.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=74, tag_tid=5, entry_tid=36646' 2026-02-19T20:32:07.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:07.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:07.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=74, tag_tid=5, entry_tid=36646' 2026-02-19T20:32:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:07.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:07.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:07.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:07.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=74, tag_tid=5, entry_tid=36646' 2026-02-19T20:32:07.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:07.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36646 2026-02-19T20:32:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 36646 2026-02-19T20:32:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:08.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:08.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:08.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:08.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:08.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:08.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:08.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:08.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:08.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:08.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:08.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:08.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:08.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:08.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:08.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:08.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=75, tag_tid=5, entry_tid=36999], [object_number=74, tag_tid=5, entry_tid=36998], [object_number=73, tag_tid=5, entry_tid=36997], [object_number=72, tag_tid=5, entry_tid=36996]]], state=connected] 2026-02-19T20:32:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=75, tag_tid=5, entry_tid=36999], [object_number=74, tag_tid=5, entry_tid=36998], [object_number=73, tag_tid=5, entry_tid=36997], [object_number=72, tag_tid=5, entry_tid=36996]]], state=connected] 2026-02-19T20:32:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=75, tag_tid=5, entry_tid=36999' 2026-02-19T20:32:08.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=75, tag_tid=5, entry_tid=36999' 2026-02-19T20:32:08.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=75, tag_tid=5, entry_tid=36999' '!=' 'object_number=74, tag_tid=5, entry_tid=36646' 2026-02-19T20:32:08.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:08.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=75, tag_tid=5, entry_tid=36999' = 'object_number=74, tag_tid=5, entry_tid=36646' 2026-02-19T20:32:08.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=75, tag_tid=5, entry_tid=36999' 2026-02-19T20:32:08.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:08.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:08.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:08.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=75, tag_tid=5, entry_tid=36999' 2026-02-19T20:32:08.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:08.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:08.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:08.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:08.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:08.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:08.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:08.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=75, tag_tid=5, entry_tid=36999' 2026-02-19T20:32:08.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:08.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:08.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36999 2026-02-19T20:32:08.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 36999 2026-02-19T20:32:08.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:08.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:08.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:08.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:08.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:08.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:08.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:08.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:08.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:08.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:08.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:08.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:08.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:08.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=73, tag_tid=5, entry_tid=37313], [object_number=72, tag_tid=5, entry_tid=37312], [object_number=75, tag_tid=5, entry_tid=37311], [object_number=74, tag_tid=5, entry_tid=37310]]], state=connected] 2026-02-19T20:32:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:08.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:08.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:08.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=73, tag_tid=5, entry_tid=37313], [object_number=72, tag_tid=5, entry_tid=37312], [object_number=75, tag_tid=5, entry_tid=37311], [object_number=74, tag_tid=5, entry_tid=37310]]], state=connected] 2026-02-19T20:32:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=73, tag_tid=5, entry_tid=37313' 2026-02-19T20:32:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=73, tag_tid=5, entry_tid=37313' 2026-02-19T20:32:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=73, tag_tid=5, entry_tid=37313' '!=' 'object_number=75, tag_tid=5, entry_tid=36999' 2026-02-19T20:32:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=73, tag_tid=5, entry_tid=37313' = 'object_number=75, tag_tid=5, entry_tid=36999' 2026-02-19T20:32:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=73, tag_tid=5, entry_tid=37313' 2026-02-19T20:32:08.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:08.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:08.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=73, tag_tid=5, entry_tid=37313' 2026-02-19T20:32:08.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:08.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:08.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=73, tag_tid=5, entry_tid=37313' 2026-02-19T20:32:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37313 2026-02-19T20:32:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 37313 2026-02-19T20:32:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:09.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:09.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:09.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:09.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:09.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:09.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=72, tag_tid=5, entry_tid=37732], [object_number=75, tag_tid=5, entry_tid=37731], [object_number=74, tag_tid=5, entry_tid=37730], [object_number=73, tag_tid=5, entry_tid=37729]]], state=connected] 2026-02-19T20:32:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=72, tag_tid=5, entry_tid=37732], [object_number=75, tag_tid=5, entry_tid=37731], [object_number=74, tag_tid=5, entry_tid=37730], [object_number=73, tag_tid=5, entry_tid=37729]]], state=connected] 2026-02-19T20:32:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=72, tag_tid=5, entry_tid=37732' 2026-02-19T20:32:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=72, tag_tid=5, entry_tid=37732' 2026-02-19T20:32:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=72, tag_tid=5, entry_tid=37732' '!=' 'object_number=73, tag_tid=5, entry_tid=37313' 2026-02-19T20:32:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=72, tag_tid=5, entry_tid=37732' = 'object_number=73, tag_tid=5, entry_tid=37313' 2026-02-19T20:32:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=72, tag_tid=5, entry_tid=37732' 2026-02-19T20:32:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=72, tag_tid=5, entry_tid=37732' 2026-02-19T20:32:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:09.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:09.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:09.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=72, tag_tid=5, entry_tid=37732' 2026-02-19T20:32:09.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:09.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37732 2026-02-19T20:32:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 37732 2026-02-19T20:32:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:09.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:09.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:09.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:09.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:09.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:09.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:09.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:09.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:09.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:09.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:09.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:09.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:09.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:09.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:09.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:09.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:09.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:09.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:09.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:09.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:09.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:09.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:09.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:09.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:09.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=75, tag_tid=5, entry_tid=38135], [object_number=74, tag_tid=5, entry_tid=38134], [object_number=73, tag_tid=5, entry_tid=38133], [object_number=72, tag_tid=5, entry_tid=38132]]], state=connected] 2026-02-19T20:32:09.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:09.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:09.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:09.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:09.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:09.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:09.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:09.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:09.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:09.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:09.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:09.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:09.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 18 2026-02-19T20:32:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=75, tag_tid=5, entry_tid=38135], [object_number=74, tag_tid=5, entry_tid=38134], [object_number=73, tag_tid=5, entry_tid=38133], [object_number=72, tag_tid=5, entry_tid=38132]]], state=connected] 2026-02-19T20:32:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=75, tag_tid=5, entry_tid=38135' 2026-02-19T20:32:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=75, tag_tid=5, entry_tid=38135' 2026-02-19T20:32:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=75, tag_tid=5, entry_tid=38135' '!=' 'object_number=72, tag_tid=5, entry_tid=37732' 2026-02-19T20:32:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=75, tag_tid=5, entry_tid=38135' = 'object_number=72, tag_tid=5, entry_tid=37732' 2026-02-19T20:32:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=75, tag_tid=5, entry_tid=38135' 2026-02-19T20:32:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:09.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:09.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:09.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=75, tag_tid=5, entry_tid=38135' 2026-02-19T20:32:09.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:09.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:09.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:09.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:09.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:09.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=75, tag_tid=5, entry_tid=38135' 2026-02-19T20:32:09.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:09.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38135 2026-02-19T20:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 38135 2026-02-19T20:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:09.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:09.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:09.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:09.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:09.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:09.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:09.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=79, tag_tid=5, entry_tid=38371], [object_number=78, tag_tid=5, entry_tid=38370], [object_number=77, tag_tid=5, entry_tid=38369], [object_number=76, tag_tid=5, entry_tid=38368]]], state=connected] 2026-02-19T20:32:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:10.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=79, tag_tid=5, entry_tid=38371], [object_number=78, tag_tid=5, entry_tid=38370], [object_number=77, tag_tid=5, entry_tid=38369], [object_number=76, tag_tid=5, entry_tid=38368]]], state=connected] 2026-02-19T20:32:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=79, tag_tid=5, entry_tid=38371' 2026-02-19T20:32:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=79, tag_tid=5, entry_tid=38371' 2026-02-19T20:32:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=79, tag_tid=5, entry_tid=38371' '!=' 'object_number=75, tag_tid=5, entry_tid=38135' 2026-02-19T20:32:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=79, tag_tid=5, entry_tid=38371' = 'object_number=75, tag_tid=5, entry_tid=38135' 2026-02-19T20:32:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=79, tag_tid=5, entry_tid=38371' 2026-02-19T20:32:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:10.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=79, tag_tid=5, entry_tid=38371' 2026-02-19T20:32:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:10.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:10.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:10.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=79, tag_tid=5, entry_tid=38371' 2026-02-19T20:32:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38371 2026-02-19T20:32:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 38371 2026-02-19T20:32:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:10.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:10.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:10.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:10.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:10.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:10.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:10.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:10.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:10.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:10.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:10.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:10.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:10.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:10.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:10.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=77, tag_tid=5, entry_tid=38649], [object_number=76, tag_tid=5, entry_tid=38648], [object_number=79, tag_tid=5, entry_tid=38647], [object_number=78, tag_tid=5, entry_tid=38646]]], state=connected] 2026-02-19T20:32:10.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:10.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:10.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:10.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=77, tag_tid=5, entry_tid=38649], [object_number=76, tag_tid=5, entry_tid=38648], [object_number=79, tag_tid=5, entry_tid=38647], [object_number=78, tag_tid=5, entry_tid=38646]]], state=connected] 2026-02-19T20:32:10.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=77, tag_tid=5, entry_tid=38649' 2026-02-19T20:32:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=77, tag_tid=5, entry_tid=38649' 2026-02-19T20:32:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=77, tag_tid=5, entry_tid=38649' '!=' 'object_number=79, tag_tid=5, entry_tid=38371' 2026-02-19T20:32:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=77, tag_tid=5, entry_tid=38649' = 'object_number=79, tag_tid=5, entry_tid=38371' 2026-02-19T20:32:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=77, tag_tid=5, entry_tid=38649' 2026-02-19T20:32:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=77, tag_tid=5, entry_tid=38649' 2026-02-19T20:32:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=77, tag_tid=5, entry_tid=38649' 2026-02-19T20:32:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:10.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38649 2026-02-19T20:32:10.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 38649 2026-02-19T20:32:10.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:10.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:10.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:10.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=39138], [object_number=77, tag_tid=5, entry_tid=39137], [object_number=76, tag_tid=5, entry_tid=39136], [object_number=79, tag_tid=5, entry_tid=39135]]], state=connected] 2026-02-19T20:32:11.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:11.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:11.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:11.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=39138], [object_number=77, tag_tid=5, entry_tid=39137], [object_number=76, tag_tid=5, entry_tid=39136], [object_number=79, tag_tid=5, entry_tid=39135]]], state=connected] 2026-02-19T20:32:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=39138' 2026-02-19T20:32:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=39138' 2026-02-19T20:32:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=39138' '!=' 'object_number=77, tag_tid=5, entry_tid=38649' 2026-02-19T20:32:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=39138' = 'object_number=77, tag_tid=5, entry_tid=38649' 2026-02-19T20:32:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=78, tag_tid=5, entry_tid=39138' 2026-02-19T20:32:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=39138' 2026-02-19T20:32:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:11.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=39138' 2026-02-19T20:32:11.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:11.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39138 2026-02-19T20:32:11.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 39138 2026-02-19T20:32:11.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:11.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:11.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:11.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:11.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:11.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:11.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:11.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:11.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:11.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:11.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:11.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:11.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:11.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:11.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:11.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:11.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:11.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:11.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:11.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:11.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:11.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:11.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=39490], [object_number=77, tag_tid=5, entry_tid=39489], [object_number=76, tag_tid=5, entry_tid=39488], [object_number=79, tag_tid=5, entry_tid=39487]]], state=connected] 2026-02-19T20:32:11.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:11.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:11.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:11.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:11.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=39490], [object_number=77, tag_tid=5, entry_tid=39489], [object_number=76, tag_tid=5, entry_tid=39488], [object_number=79, tag_tid=5, entry_tid=39487]]], state=connected] 2026-02-19T20:32:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=39490' 2026-02-19T20:32:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=39490' 2026-02-19T20:32:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=39490' '!=' 'object_number=78, tag_tid=5, entry_tid=39138' 2026-02-19T20:32:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=39490' = 'object_number=78, tag_tid=5, entry_tid=39138' 2026-02-19T20:32:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=78, tag_tid=5, entry_tid=39490' 2026-02-19T20:32:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=39490' 2026-02-19T20:32:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:11.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=39490' 2026-02-19T20:32:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39490 2026-02-19T20:32:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 39490 2026-02-19T20:32:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:11.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:11.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:11.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:11.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:11.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:11.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:11.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:11.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:11.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:11.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:11.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:11.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=79, tag_tid=5, entry_tid=39871], [object_number=78, tag_tid=5, entry_tid=39870], [object_number=77, tag_tid=5, entry_tid=39869], [object_number=76, tag_tid=5, entry_tid=39868]]], state=connected] 2026-02-19T20:32:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=79, tag_tid=5, entry_tid=39871], [object_number=78, tag_tid=5, entry_tid=39870], [object_number=77, tag_tid=5, entry_tid=39869], [object_number=76, tag_tid=5, entry_tid=39868]]], state=connected] 2026-02-19T20:32:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=79, tag_tid=5, entry_tid=39871' 2026-02-19T20:32:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=79, tag_tid=5, entry_tid=39871' 2026-02-19T20:32:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=79, tag_tid=5, entry_tid=39871' '!=' 'object_number=78, tag_tid=5, entry_tid=39490' 2026-02-19T20:32:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=79, tag_tid=5, entry_tid=39871' = 'object_number=78, tag_tid=5, entry_tid=39490' 2026-02-19T20:32:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=79, tag_tid=5, entry_tid=39871' 2026-02-19T20:32:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=79, tag_tid=5, entry_tid=39871' 2026-02-19T20:32:11.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:11.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:11.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:11.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:11.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:11.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=79, tag_tid=5, entry_tid=39871' 2026-02-19T20:32:11.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:11.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39871 2026-02-19T20:32:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 39871 2026-02-19T20:32:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:12.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:12.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:12.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:12.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:12.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:12.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:12.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:12.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:12.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:12.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:12.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:12.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:12.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:12.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:12.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:12.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:12.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:12.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:12.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=79, tag_tid=5, entry_tid=40235], [object_number=78, tag_tid=5, entry_tid=40234], [object_number=77, tag_tid=5, entry_tid=40233], [object_number=76, tag_tid=5, entry_tid=40232]]], state=connected] 2026-02-19T20:32:12.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=79, tag_tid=5, entry_tid=40235], [object_number=78, tag_tid=5, entry_tid=40234], [object_number=77, tag_tid=5, entry_tid=40233], [object_number=76, tag_tid=5, entry_tid=40232]]], state=connected] 2026-02-19T20:32:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=79, tag_tid=5, entry_tid=40235' 2026-02-19T20:32:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=79, tag_tid=5, entry_tid=40235' 2026-02-19T20:32:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=79, tag_tid=5, entry_tid=40235' '!=' 'object_number=79, tag_tid=5, entry_tid=39871' 2026-02-19T20:32:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=79, tag_tid=5, entry_tid=40235' = 'object_number=79, tag_tid=5, entry_tid=39871' 2026-02-19T20:32:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=79, tag_tid=5, entry_tid=40235' 2026-02-19T20:32:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:12.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=79, tag_tid=5, entry_tid=40235' 2026-02-19T20:32:12.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=79, tag_tid=5, entry_tid=40235' 2026-02-19T20:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40235 2026-02-19T20:32:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 40235 2026-02-19T20:32:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:12.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:12.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:12.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:12.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:12.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:12.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:12.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:12.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:12.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:12.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:12.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:12.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:12.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:12.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:12.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:12.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:12.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:12.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:12.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=40598], [object_number=77, tag_tid=5, entry_tid=40597], [object_number=76, tag_tid=5, entry_tid=40596], [object_number=79, tag_tid=5, entry_tid=40595]]], state=connected] 2026-02-19T20:32:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:12.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:12.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:12.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:12.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:12.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:12.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:12.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:12.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:12.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:12.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:12.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:12.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:12.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:12.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:12.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=40598], [object_number=77, tag_tid=5, entry_tid=40597], [object_number=76, tag_tid=5, entry_tid=40596], [object_number=79, tag_tid=5, entry_tid=40595]]], state=connected] 2026-02-19T20:32:12.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=40598' 2026-02-19T20:32:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=40598' 2026-02-19T20:32:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=40598' '!=' 'object_number=79, tag_tid=5, entry_tid=40235' 2026-02-19T20:32:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:12.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=40598' = 'object_number=79, tag_tid=5, entry_tid=40235' 2026-02-19T20:32:12.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=78, tag_tid=5, entry_tid=40598' 2026-02-19T20:32:12.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:12.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:12.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:12.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:12.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=40598' 2026-02-19T20:32:12.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:12.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:12.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:12.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:12.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=40598' 2026-02-19T20:32:12.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:12.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40598 2026-02-19T20:32:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 40598 2026-02-19T20:32:12.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:12.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:12.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:13.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:13.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:13.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:13.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:13.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:13.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:13.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:13.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:13.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:13.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:13.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:13.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:13.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:13.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=77, tag_tid=5, entry_tid=40953], [object_number=76, tag_tid=5, entry_tid=40952], [object_number=79, tag_tid=5, entry_tid=40951], [object_number=78, tag_tid=5, entry_tid=40950]]], state=connected] 2026-02-19T20:32:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:13.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:13.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=77, tag_tid=5, entry_tid=40953], [object_number=76, tag_tid=5, entry_tid=40952], [object_number=79, tag_tid=5, entry_tid=40951], [object_number=78, tag_tid=5, entry_tid=40950]]], state=connected] 2026-02-19T20:32:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=77, tag_tid=5, entry_tid=40953' 2026-02-19T20:32:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=77, tag_tid=5, entry_tid=40953' 2026-02-19T20:32:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=77, tag_tid=5, entry_tid=40953' '!=' 'object_number=78, tag_tid=5, entry_tid=40598' 2026-02-19T20:32:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=77, tag_tid=5, entry_tid=40953' = 'object_number=78, tag_tid=5, entry_tid=40598' 2026-02-19T20:32:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=77, tag_tid=5, entry_tid=40953' 2026-02-19T20:32:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=77, tag_tid=5, entry_tid=40953' 2026-02-19T20:32:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=77, tag_tid=5, entry_tid=40953' 2026-02-19T20:32:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40953 2026-02-19T20:32:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 40953 2026-02-19T20:32:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:13.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:13.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:13.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:13.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:13.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:13.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:13.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:13.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:13.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:13.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:13.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=41274], [object_number=77, tag_tid=5, entry_tid=41273], [object_number=76, tag_tid=5, entry_tid=41272], [object_number=79, tag_tid=5, entry_tid=41271]]], state=connected] 2026-02-19T20:32:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:13.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:13.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:13.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:13.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:13.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:13.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:13.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:13.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:13.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:13.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:13.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:13.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=41274], [object_number=77, tag_tid=5, entry_tid=41273], [object_number=76, tag_tid=5, entry_tid=41272], [object_number=79, tag_tid=5, entry_tid=41271]]], state=connected] 2026-02-19T20:32:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:13.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=41274' 2026-02-19T20:32:13.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=41274' 2026-02-19T20:32:13.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=41274' '!=' 'object_number=77, tag_tid=5, entry_tid=40953' 2026-02-19T20:32:13.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:13.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=41274' = 'object_number=77, tag_tid=5, entry_tid=40953' 2026-02-19T20:32:13.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=78, tag_tid=5, entry_tid=41274' 2026-02-19T20:32:13.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:13.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:13.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:13.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=41274' 2026-02-19T20:32:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:13.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:13.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:13.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=41274' 2026-02-19T20:32:13.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:13.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:13.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41274 2026-02-19T20:32:13.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 41274 2026-02-19T20:32:13.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:13.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:13.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:13.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:13.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:13.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:13.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:13.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:13.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:13.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:13.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:13.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:13.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:13.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:13.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:13.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:13.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:13.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:13.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:13.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:13.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:13.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:13.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:13.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:13.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:13.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=76, tag_tid=5, entry_tid=41572], [object_number=79, tag_tid=5, entry_tid=41571], [object_number=78, tag_tid=5, entry_tid=41570], [object_number=77, tag_tid=5, entry_tid=41569]]], state=connected] 2026-02-19T20:32:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:14.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=76, tag_tid=5, entry_tid=41572], [object_number=79, tag_tid=5, entry_tid=41571], [object_number=78, tag_tid=5, entry_tid=41570], [object_number=77, tag_tid=5, entry_tid=41569]]], state=connected] 2026-02-19T20:32:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=76, tag_tid=5, entry_tid=41572' 2026-02-19T20:32:14.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=76, tag_tid=5, entry_tid=41572' 2026-02-19T20:32:14.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=76, tag_tid=5, entry_tid=41572' '!=' 'object_number=78, tag_tid=5, entry_tid=41274' 2026-02-19T20:32:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=76, tag_tid=5, entry_tid=41572' = 'object_number=78, tag_tid=5, entry_tid=41274' 2026-02-19T20:32:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=76, tag_tid=5, entry_tid=41572' 2026-02-19T20:32:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=76, tag_tid=5, entry_tid=41572' 2026-02-19T20:32:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:14.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=76, tag_tid=5, entry_tid=41572' 2026-02-19T20:32:14.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:14.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41572 2026-02-19T20:32:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 41572 2026-02-19T20:32:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:14.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:14.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:14.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:14.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:14.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:14.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=41870], [object_number=77, tag_tid=5, entry_tid=41869], [object_number=76, tag_tid=5, entry_tid=41868], [object_number=79, tag_tid=5, entry_tid=41867]]], state=connected] 2026-02-19T20:32:14.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:14.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:14.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=41870], [object_number=77, tag_tid=5, entry_tid=41869], [object_number=76, tag_tid=5, entry_tid=41868], [object_number=79, tag_tid=5, entry_tid=41867]]], state=connected] 2026-02-19T20:32:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=41870' 2026-02-19T20:32:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=41870' 2026-02-19T20:32:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=41870' '!=' 'object_number=76, tag_tid=5, entry_tid=41572' 2026-02-19T20:32:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=41870' = 'object_number=76, tag_tid=5, entry_tid=41572' 2026-02-19T20:32:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=78, tag_tid=5, entry_tid=41870' 2026-02-19T20:32:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=41870' 2026-02-19T20:32:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:14.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=41870' 2026-02-19T20:32:14.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:14.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41870 2026-02-19T20:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 41870 2026-02-19T20:32:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:14.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:14.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:14.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:14.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:14.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:14.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:14.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:14.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:14.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:14.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:14.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:14.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:14.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:14.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:14.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:14.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:14.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:14.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:14.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:14.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=77, tag_tid=5, entry_tid=42157], [object_number=76, tag_tid=5, entry_tid=42156], [object_number=79, tag_tid=5, entry_tid=42155], [object_number=78, tag_tid=5, entry_tid=42154]]], state=connected] 2026-02-19T20:32:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=77, tag_tid=5, entry_tid=42157], [object_number=76, tag_tid=5, entry_tid=42156], [object_number=79, tag_tid=5, entry_tid=42155], [object_number=78, tag_tid=5, entry_tid=42154]]], state=connected] 2026-02-19T20:32:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=77, tag_tid=5, entry_tid=42157' 2026-02-19T20:32:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=77, tag_tid=5, entry_tid=42157' 2026-02-19T20:32:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=77, tag_tid=5, entry_tid=42157' '!=' 'object_number=78, tag_tid=5, entry_tid=41870' 2026-02-19T20:32:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=77, tag_tid=5, entry_tid=42157' = 'object_number=78, tag_tid=5, entry_tid=41870' 2026-02-19T20:32:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=77, tag_tid=5, entry_tid=42157' 2026-02-19T20:32:14.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:14.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:14.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:14.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=77, tag_tid=5, entry_tid=42157' 2026-02-19T20:32:14.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:14.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:14.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=77, tag_tid=5, entry_tid=42157' 2026-02-19T20:32:14.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:14.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:14.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42157 2026-02-19T20:32:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 42157 2026-02-19T20:32:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:15.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:15.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:15.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:15.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:15.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:15.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:15.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:15.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:15.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:15.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:15.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:15.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:15.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=42388], [object_number=83, tag_tid=5, entry_tid=42387], [object_number=78, tag_tid=5, entry_tid=42386], [object_number=77, tag_tid=5, entry_tid=42385]]], state=connected] 2026-02-19T20:32:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 19 2026-02-19T20:32:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=42388], [object_number=83, tag_tid=5, entry_tid=42387], [object_number=78, tag_tid=5, entry_tid=42386], [object_number=77, tag_tid=5, entry_tid=42385]]], state=connected] 2026-02-19T20:32:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=80, tag_tid=5, entry_tid=42388' 2026-02-19T20:32:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=80, tag_tid=5, entry_tid=42388' 2026-02-19T20:32:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=42388' '!=' 'object_number=77, tag_tid=5, entry_tid=42157' 2026-02-19T20:32:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=42388' = 'object_number=77, tag_tid=5, entry_tid=42157' 2026-02-19T20:32:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=80, tag_tid=5, entry_tid=42388' 2026-02-19T20:32:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:15.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=42388' 2026-02-19T20:32:15.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:15.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:15.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:15.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:15.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:15.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:15.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:15.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=42388' 2026-02-19T20:32:15.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:15.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42388 2026-02-19T20:32:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 42388 2026-02-19T20:32:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:15.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:15.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:15.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:15.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:15.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:15.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:15.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:15.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:15.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:15.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:15.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=42680], [object_number=83, tag_tid=5, entry_tid=42679], [object_number=82, tag_tid=5, entry_tid=42678], [object_number=81, tag_tid=5, entry_tid=42677]]], state=connected] 2026-02-19T20:32:15.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:15.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:15.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:15.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:15.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:15.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:15.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=42680], [object_number=83, tag_tid=5, entry_tid=42679], [object_number=82, tag_tid=5, entry_tid=42678], [object_number=81, tag_tid=5, entry_tid=42677]]], state=connected] 2026-02-19T20:32:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=80, tag_tid=5, entry_tid=42680' 2026-02-19T20:32:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=80, tag_tid=5, entry_tid=42680' 2026-02-19T20:32:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=42680' '!=' 'object_number=80, tag_tid=5, entry_tid=42388' 2026-02-19T20:32:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=42680' = 'object_number=80, tag_tid=5, entry_tid=42388' 2026-02-19T20:32:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=80, tag_tid=5, entry_tid=42680' 2026-02-19T20:32:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=42680' 2026-02-19T20:32:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=42680' 2026-02-19T20:32:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42680 2026-02-19T20:32:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 42680 2026-02-19T20:32:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:16.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:16.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:16.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:16.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:16.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:16.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:16.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:16.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=83, tag_tid=5, entry_tid=43019], [object_number=82, tag_tid=5, entry_tid=43018], [object_number=81, tag_tid=5, entry_tid=43017], [object_number=80, tag_tid=5, entry_tid=43016]]], state=connected] 2026-02-19T20:32:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:16.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:16.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:16.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:16.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:16.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=83, tag_tid=5, entry_tid=43019], [object_number=82, tag_tid=5, entry_tid=43018], [object_number=81, tag_tid=5, entry_tid=43017], [object_number=80, tag_tid=5, entry_tid=43016]]], state=connected] 2026-02-19T20:32:16.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=83, tag_tid=5, entry_tid=43019' 2026-02-19T20:32:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=83, tag_tid=5, entry_tid=43019' 2026-02-19T20:32:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=83, tag_tid=5, entry_tid=43019' '!=' 'object_number=80, tag_tid=5, entry_tid=42680' 2026-02-19T20:32:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=83, tag_tid=5, entry_tid=43019' = 'object_number=80, tag_tid=5, entry_tid=42680' 2026-02-19T20:32:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=83, tag_tid=5, entry_tid=43019' 2026-02-19T20:32:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=83, tag_tid=5, entry_tid=43019' 2026-02-19T20:32:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:16.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:16.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=83, tag_tid=5, entry_tid=43019' 2026-02-19T20:32:16.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:16.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43019 2026-02-19T20:32:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 43019 2026-02-19T20:32:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:16.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:16.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:16.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:16.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:16.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:16.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:16.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:16.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:16.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:16.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:16.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:16.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:16.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:16.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:16.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:16.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:16.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:16.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:16.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:16.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:16.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:16.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=81, tag_tid=5, entry_tid=43325], [object_number=80, tag_tid=5, entry_tid=43324], [object_number=83, tag_tid=5, entry_tid=43323], [object_number=82, tag_tid=5, entry_tid=43322]]], state=connected] 2026-02-19T20:32:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:16.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:16.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:16.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:16.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:16.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:16.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:16.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:16.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:16.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:16.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=81, tag_tid=5, entry_tid=43325], [object_number=80, tag_tid=5, entry_tid=43324], [object_number=83, tag_tid=5, entry_tid=43323], [object_number=82, tag_tid=5, entry_tid=43322]]], state=connected] 2026-02-19T20:32:16.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=81, tag_tid=5, entry_tid=43325' 2026-02-19T20:32:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=81, tag_tid=5, entry_tid=43325' 2026-02-19T20:32:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=81, tag_tid=5, entry_tid=43325' '!=' 'object_number=83, tag_tid=5, entry_tid=43019' 2026-02-19T20:32:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=81, tag_tid=5, entry_tid=43325' = 'object_number=83, tag_tid=5, entry_tid=43019' 2026-02-19T20:32:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=81, tag_tid=5, entry_tid=43325' 2026-02-19T20:32:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:16.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=81, tag_tid=5, entry_tid=43325' 2026-02-19T20:32:16.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:16.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:16.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:16.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=81, tag_tid=5, entry_tid=43325' 2026-02-19T20:32:16.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:16.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:16.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43325 2026-02-19T20:32:16.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 43325 2026-02-19T20:32:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:16.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:16.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:16.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:16.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:16.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:16.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:16.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:16.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:16.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:16.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:16.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:16.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:16.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:16.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:16.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=43652], [object_number=83, tag_tid=5, entry_tid=43651], [object_number=82, tag_tid=5, entry_tid=43650], [object_number=81, tag_tid=5, entry_tid=43649]]], state=connected] 2026-02-19T20:32:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:17.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:17.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:17.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:17.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=43652], [object_number=83, tag_tid=5, entry_tid=43651], [object_number=82, tag_tid=5, entry_tid=43650], [object_number=81, tag_tid=5, entry_tid=43649]]], state=connected] 2026-02-19T20:32:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=80, tag_tid=5, entry_tid=43652' 2026-02-19T20:32:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=80, tag_tid=5, entry_tid=43652' 2026-02-19T20:32:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=43652' '!=' 'object_number=81, tag_tid=5, entry_tid=43325' 2026-02-19T20:32:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=43652' = 'object_number=81, tag_tid=5, entry_tid=43325' 2026-02-19T20:32:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=80, tag_tid=5, entry_tid=43652' 2026-02-19T20:32:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=43652' 2026-02-19T20:32:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=43652' 2026-02-19T20:32:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43652 2026-02-19T20:32:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 43652 2026-02-19T20:32:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:17.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:17.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:17.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:17.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:17.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:17.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:17.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:17.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:17.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:17.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:17.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:17.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:17.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=44024], [object_number=83, tag_tid=5, entry_tid=44023], [object_number=82, tag_tid=5, entry_tid=44022], [object_number=81, tag_tid=5, entry_tid=44021]]], state=connected] 2026-02-19T20:32:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:17.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:17.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=44024], [object_number=83, tag_tid=5, entry_tid=44023], [object_number=82, tag_tid=5, entry_tid=44022], [object_number=81, tag_tid=5, entry_tid=44021]]], state=connected] 2026-02-19T20:32:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=80, tag_tid=5, entry_tid=44024' 2026-02-19T20:32:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=80, tag_tid=5, entry_tid=44024' 2026-02-19T20:32:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=44024' '!=' 'object_number=80, tag_tid=5, entry_tid=43652' 2026-02-19T20:32:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=44024' = 'object_number=80, tag_tid=5, entry_tid=43652' 2026-02-19T20:32:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=80, tag_tid=5, entry_tid=44024' 2026-02-19T20:32:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=44024' 2026-02-19T20:32:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=44024' 2026-02-19T20:32:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:17.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44024 2026-02-19T20:32:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 44024 2026-02-19T20:32:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:17.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:17.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:17.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:17.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:17.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:17.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:17.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:17.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:17.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:17.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:17.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:17.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:17.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:17.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:17.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:17.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:17.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=44328], [object_number=83, tag_tid=5, entry_tid=44327], [object_number=82, tag_tid=5, entry_tid=44326], [object_number=81, tag_tid=5, entry_tid=44325]]], state=connected] 2026-02-19T20:32:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:17.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:17.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:17.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:17.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=44328], [object_number=83, tag_tid=5, entry_tid=44327], [object_number=82, tag_tid=5, entry_tid=44326], [object_number=81, tag_tid=5, entry_tid=44325]]], state=connected] 2026-02-19T20:32:17.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=80, tag_tid=5, entry_tid=44328' 2026-02-19T20:32:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=80, tag_tid=5, entry_tid=44328' 2026-02-19T20:32:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=44328' '!=' 'object_number=80, tag_tid=5, entry_tid=44024' 2026-02-19T20:32:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=44328' = 'object_number=80, tag_tid=5, entry_tid=44024' 2026-02-19T20:32:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=80, tag_tid=5, entry_tid=44328' 2026-02-19T20:32:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:17.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=44328' 2026-02-19T20:32:17.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:17.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:17.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=44328' 2026-02-19T20:32:17.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:17.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44328 2026-02-19T20:32:17.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 44328 2026-02-19T20:32:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:18.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:18.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:18.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:18.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:18.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:18.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:18.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:18.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=44680], [object_number=83, tag_tid=5, entry_tid=44679], [object_number=82, tag_tid=5, entry_tid=44678], [object_number=81, tag_tid=5, entry_tid=44677]]], state=connected] 2026-02-19T20:32:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:18.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:18.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:18.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:18.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:18.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:18.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:18.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=44680], [object_number=83, tag_tid=5, entry_tid=44679], [object_number=82, tag_tid=5, entry_tid=44678], [object_number=81, tag_tid=5, entry_tid=44677]]], state=connected] 2026-02-19T20:32:18.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=80, tag_tid=5, entry_tid=44680' 2026-02-19T20:32:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=80, tag_tid=5, entry_tid=44680' 2026-02-19T20:32:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=44680' '!=' 'object_number=80, tag_tid=5, entry_tid=44328' 2026-02-19T20:32:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=44680' = 'object_number=80, tag_tid=5, entry_tid=44328' 2026-02-19T20:32:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=80, tag_tid=5, entry_tid=44680' 2026-02-19T20:32:18.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:18.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:18.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:18.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:18.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=44680' 2026-02-19T20:32:18.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:18.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:18.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=44680' 2026-02-19T20:32:18.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:18.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:18.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44680 2026-02-19T20:32:18.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 44680 2026-02-19T20:32:18.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:18.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:18.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:18.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:18.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:18.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:18.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:18.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:18.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=45020], [object_number=83, tag_tid=5, entry_tid=45019], [object_number=82, tag_tid=5, entry_tid=45018], [object_number=81, tag_tid=5, entry_tid=45017]]], state=connected] 2026-02-19T20:32:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:18.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:18.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:18.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:18.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:18.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:18.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:18.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:18.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=45020], [object_number=83, tag_tid=5, entry_tid=45019], [object_number=82, tag_tid=5, entry_tid=45018], [object_number=81, tag_tid=5, entry_tid=45017]]], state=connected] 2026-02-19T20:32:18.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=80, tag_tid=5, entry_tid=45020' 2026-02-19T20:32:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=80, tag_tid=5, entry_tid=45020' 2026-02-19T20:32:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=45020' '!=' 'object_number=80, tag_tid=5, entry_tid=44680' 2026-02-19T20:32:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=45020' = 'object_number=80, tag_tid=5, entry_tid=44680' 2026-02-19T20:32:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=80, tag_tid=5, entry_tid=45020' 2026-02-19T20:32:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:18.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=45020' 2026-02-19T20:32:18.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:18.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:18.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:18.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:18.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:18.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=45020' 2026-02-19T20:32:18.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:18.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45020 2026-02-19T20:32:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 45020 2026-02-19T20:32:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:18.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:18.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:18.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:18.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:18.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:18.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:18.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:19.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:19.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=82, tag_tid=5, entry_tid=45338], [object_number=81, tag_tid=5, entry_tid=45337], [object_number=80, tag_tid=5, entry_tid=45336], [object_number=83, tag_tid=5, entry_tid=45335]]], state=connected] 2026-02-19T20:32:19.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:19.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=82, tag_tid=5, entry_tid=45338], [object_number=81, tag_tid=5, entry_tid=45337], [object_number=80, tag_tid=5, entry_tid=45336], [object_number=83, tag_tid=5, entry_tid=45335]]], state=connected] 2026-02-19T20:32:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=82, tag_tid=5, entry_tid=45338' 2026-02-19T20:32:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=82, tag_tid=5, entry_tid=45338' 2026-02-19T20:32:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=82, tag_tid=5, entry_tid=45338' '!=' 'object_number=80, tag_tid=5, entry_tid=45020' 2026-02-19T20:32:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=82, tag_tid=5, entry_tid=45338' = 'object_number=80, tag_tid=5, entry_tid=45020' 2026-02-19T20:32:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=82, tag_tid=5, entry_tid=45338' 2026-02-19T20:32:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=82, tag_tid=5, entry_tid=45338' 2026-02-19T20:32:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=82, tag_tid=5, entry_tid=45338' 2026-02-19T20:32:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:19.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45338 2026-02-19T20:32:19.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 45338 2026-02-19T20:32:19.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:19.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:19.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:19.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:19.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:19.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:19.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:19.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:19.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=82, tag_tid=5, entry_tid=45674], [object_number=81, tag_tid=5, entry_tid=45673], [object_number=80, tag_tid=5, entry_tid=45672], [object_number=83, tag_tid=5, entry_tid=45671]]], state=connected] 2026-02-19T20:32:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:19.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=82, tag_tid=5, entry_tid=45674], [object_number=81, tag_tid=5, entry_tid=45673], [object_number=80, tag_tid=5, entry_tid=45672], [object_number=83, tag_tid=5, entry_tid=45671]]], state=connected] 2026-02-19T20:32:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=82, tag_tid=5, entry_tid=45674' 2026-02-19T20:32:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=82, tag_tid=5, entry_tid=45674' 2026-02-19T20:32:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=82, tag_tid=5, entry_tid=45674' '!=' 'object_number=82, tag_tid=5, entry_tid=45338' 2026-02-19T20:32:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=82, tag_tid=5, entry_tid=45674' = 'object_number=82, tag_tid=5, entry_tid=45338' 2026-02-19T20:32:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=82, tag_tid=5, entry_tid=45674' 2026-02-19T20:32:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=82, tag_tid=5, entry_tid=45674' 2026-02-19T20:32:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:19.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:19.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:19.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=82, tag_tid=5, entry_tid=45674' 2026-02-19T20:32:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:19.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45674 2026-02-19T20:32:19.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 45674 2026-02-19T20:32:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:19.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:19.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:19.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:19.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:19.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:19.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:19.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:19.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:19.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:19.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:19.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:19.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:19.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:19.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=81, tag_tid=5, entry_tid=45993], [object_number=80, tag_tid=5, entry_tid=45992], [object_number=83, tag_tid=5, entry_tid=45991], [object_number=82, tag_tid=5, entry_tid=45990]]], state=connected] 2026-02-19T20:32:19.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:19.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:19.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:19.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:19.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=81, tag_tid=5, entry_tid=45993], [object_number=80, tag_tid=5, entry_tid=45992], [object_number=83, tag_tid=5, entry_tid=45991], [object_number=82, tag_tid=5, entry_tid=45990]]], state=connected] 2026-02-19T20:32:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=81, tag_tid=5, entry_tid=45993' 2026-02-19T20:32:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=81, tag_tid=5, entry_tid=45993' 2026-02-19T20:32:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=81, tag_tid=5, entry_tid=45993' '!=' 'object_number=82, tag_tid=5, entry_tid=45674' 2026-02-19T20:32:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=81, tag_tid=5, entry_tid=45993' = 'object_number=82, tag_tid=5, entry_tid=45674' 2026-02-19T20:32:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=81, tag_tid=5, entry_tid=45993' 2026-02-19T20:32:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=81, tag_tid=5, entry_tid=45993' 2026-02-19T20:32:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=81, tag_tid=5, entry_tid=45993' 2026-02-19T20:32:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45993 2026-02-19T20:32:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 45993 2026-02-19T20:32:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:20.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:20.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:20.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:20.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:20.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:20.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:20.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:20.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:20.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:20.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:20.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=83, tag_tid=5, entry_tid=46159], [object_number=82, tag_tid=5, entry_tid=46158], [object_number=81, tag_tid=5, entry_tid=46157], [object_number=80, tag_tid=5, entry_tid=46156]]], state=connected] 2026-02-19T20:32:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:20.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:20.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:20.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:20.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:20.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:20.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:20.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:20.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:20.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:20.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:20.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:20.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=83, tag_tid=5, entry_tid=46159], [object_number=82, tag_tid=5, entry_tid=46158], [object_number=81, tag_tid=5, entry_tid=46157], [object_number=80, tag_tid=5, entry_tid=46156]]], state=connected] 2026-02-19T20:32:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=83, tag_tid=5, entry_tid=46159' 2026-02-19T20:32:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=83, tag_tid=5, entry_tid=46159' 2026-02-19T20:32:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=83, tag_tid=5, entry_tid=46159' '!=' 'object_number=81, tag_tid=5, entry_tid=45993' 2026-02-19T20:32:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=83, tag_tid=5, entry_tid=46159' = 'object_number=81, tag_tid=5, entry_tid=45993' 2026-02-19T20:32:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=83, tag_tid=5, entry_tid=46159' 2026-02-19T20:32:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:20.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=83, tag_tid=5, entry_tid=46159' 2026-02-19T20:32:20.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:20.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=83, tag_tid=5, entry_tid=46159' 2026-02-19T20:32:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46159 2026-02-19T20:32:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 46159 2026-02-19T20:32:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:20.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:20.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:20.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:20.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:20.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:20.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:20.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:20.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:20.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:20.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:20.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=46480], [object_number=83, tag_tid=5, entry_tid=46479], [object_number=82, tag_tid=5, entry_tid=46478], [object_number=81, tag_tid=5, entry_tid=46477]]], state=connected] 2026-02-19T20:32:20.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 20 2026-02-19T20:32:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:20.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=80, tag_tid=5, entry_tid=46480], [object_number=83, tag_tid=5, entry_tid=46479], [object_number=82, tag_tid=5, entry_tid=46478], [object_number=81, tag_tid=5, entry_tid=46477]]], state=connected] 2026-02-19T20:32:20.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=80, tag_tid=5, entry_tid=46480' 2026-02-19T20:32:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=80, tag_tid=5, entry_tid=46480' 2026-02-19T20:32:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=46480' '!=' 'object_number=83, tag_tid=5, entry_tid=46159' 2026-02-19T20:32:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=80, tag_tid=5, entry_tid=46480' = 'object_number=83, tag_tid=5, entry_tid=46159' 2026-02-19T20:32:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=80, tag_tid=5, entry_tid=46480' 2026-02-19T20:32:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:20.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=46480' 2026-02-19T20:32:20.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:20.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:20.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=80, tag_tid=5, entry_tid=46480' 2026-02-19T20:32:20.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:20.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46480 2026-02-19T20:32:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 46480 2026-02-19T20:32:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=46741], [object_number=84, tag_tid=5, entry_tid=46740], [object_number=87, tag_tid=5, entry_tid=46739], [object_number=86, tag_tid=5, entry_tid=46738]]], state=connected] 2026-02-19T20:32:21.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:21.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:21.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:21.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:21.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=46741], [object_number=84, tag_tid=5, entry_tid=46740], [object_number=87, tag_tid=5, entry_tid=46739], [object_number=86, tag_tid=5, entry_tid=46738]]], state=connected] 2026-02-19T20:32:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=46741' 2026-02-19T20:32:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=46741' 2026-02-19T20:32:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=46741' '!=' 'object_number=80, tag_tid=5, entry_tid=46480' 2026-02-19T20:32:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=46741' = 'object_number=80, tag_tid=5, entry_tid=46480' 2026-02-19T20:32:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=85, tag_tid=5, entry_tid=46741' 2026-02-19T20:32:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=46741' 2026-02-19T20:32:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:21.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:21.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=46741' 2026-02-19T20:32:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46741 2026-02-19T20:32:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 46741 2026-02-19T20:32:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:21.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:21.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:21.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:21.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:21.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:21.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:21.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:21.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:21.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:21.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:21.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:21.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=47113], [object_number=84, tag_tid=5, entry_tid=47112], [object_number=87, tag_tid=5, entry_tid=47111], [object_number=86, tag_tid=5, entry_tid=47110]]], state=connected] 2026-02-19T20:32:21.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=47113], [object_number=84, tag_tid=5, entry_tid=47112], [object_number=87, tag_tid=5, entry_tid=47111], [object_number=86, tag_tid=5, entry_tid=47110]]], state=connected] 2026-02-19T20:32:21.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:21.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=47113' 2026-02-19T20:32:21.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=47113' 2026-02-19T20:32:21.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=47113' '!=' 'object_number=85, tag_tid=5, entry_tid=46741' 2026-02-19T20:32:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=47113' = 'object_number=85, tag_tid=5, entry_tid=46741' 2026-02-19T20:32:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=85, tag_tid=5, entry_tid=47113' 2026-02-19T20:32:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:21.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:21.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:21.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:21.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=47113' 2026-02-19T20:32:21.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:21.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:21.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:21.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=47113' 2026-02-19T20:32:21.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:21.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:21.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47113 2026-02-19T20:32:21.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 47113 2026-02-19T20:32:21.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:21.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:21.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:21.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:21.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:21.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:21.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:21.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:21.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:21.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:21.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:21.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:21.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:21.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:21.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:21.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:21.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:21.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:21.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:21.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:21.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:21.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:21.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:21.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:21.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:21.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:22.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:22.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:22.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:22.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:22.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:22.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:22.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=47408], [object_number=87, tag_tid=5, entry_tid=47407], [object_number=86, tag_tid=5, entry_tid=47406], [object_number=85, tag_tid=5, entry_tid=47405]]], state=connected] 2026-02-19T20:32:22.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:22.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:22.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:22.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:22.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:22.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:22.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:22.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=47408], [object_number=87, tag_tid=5, entry_tid=47407], [object_number=86, tag_tid=5, entry_tid=47406], [object_number=85, tag_tid=5, entry_tid=47405]]], state=connected] 2026-02-19T20:32:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=84, tag_tid=5, entry_tid=47408' 2026-02-19T20:32:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=84, tag_tid=5, entry_tid=47408' 2026-02-19T20:32:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=47408' '!=' 'object_number=85, tag_tid=5, entry_tid=47113' 2026-02-19T20:32:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=47408' = 'object_number=85, tag_tid=5, entry_tid=47113' 2026-02-19T20:32:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=84, tag_tid=5, entry_tid=47408' 2026-02-19T20:32:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=47408' 2026-02-19T20:32:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=47408' 2026-02-19T20:32:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47408 2026-02-19T20:32:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 47408 2026-02-19T20:32:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:22.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=87, tag_tid=5, entry_tid=47731], [object_number=86, tag_tid=5, entry_tid=47730], [object_number=85, tag_tid=5, entry_tid=47729], [object_number=84, tag_tid=5, entry_tid=47728]]], state=connected] 2026-02-19T20:32:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:22.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:22.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=87, tag_tid=5, entry_tid=47731], [object_number=86, tag_tid=5, entry_tid=47730], [object_number=85, tag_tid=5, entry_tid=47729], [object_number=84, tag_tid=5, entry_tid=47728]]], state=connected] 2026-02-19T20:32:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=87, tag_tid=5, entry_tid=47731' 2026-02-19T20:32:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=87, tag_tid=5, entry_tid=47731' 2026-02-19T20:32:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=87, tag_tid=5, entry_tid=47731' '!=' 'object_number=84, tag_tid=5, entry_tid=47408' 2026-02-19T20:32:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=87, tag_tid=5, entry_tid=47731' = 'object_number=84, tag_tid=5, entry_tid=47408' 2026-02-19T20:32:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=87, tag_tid=5, entry_tid=47731' 2026-02-19T20:32:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:22.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=87, tag_tid=5, entry_tid=47731' 2026-02-19T20:32:22.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:22.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:22.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:22.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:22.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:22.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=87, tag_tid=5, entry_tid=47731' 2026-02-19T20:32:22.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:22.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47731 2026-02-19T20:32:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 47731 2026-02-19T20:32:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:22.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=86, tag_tid=5, entry_tid=48022], [object_number=85, tag_tid=5, entry_tid=48021], [object_number=84, tag_tid=5, entry_tid=48020], [object_number=87, tag_tid=5, entry_tid=48019]]], state=connected] 2026-02-19T20:32:22.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:22.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:22.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:22.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:22.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:22.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:22.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:22.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=86, tag_tid=5, entry_tid=48022], [object_number=85, tag_tid=5, entry_tid=48021], [object_number=84, tag_tid=5, entry_tid=48020], [object_number=87, tag_tid=5, entry_tid=48019]]], state=connected] 2026-02-19T20:32:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=86, tag_tid=5, entry_tid=48022' 2026-02-19T20:32:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=86, tag_tid=5, entry_tid=48022' 2026-02-19T20:32:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=86, tag_tid=5, entry_tid=48022' '!=' 'object_number=87, tag_tid=5, entry_tid=47731' 2026-02-19T20:32:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=86, tag_tid=5, entry_tid=48022' = 'object_number=87, tag_tid=5, entry_tid=47731' 2026-02-19T20:32:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=86, tag_tid=5, entry_tid=48022' 2026-02-19T20:32:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=86, tag_tid=5, entry_tid=48022' 2026-02-19T20:32:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:22.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:22.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:22.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=86, tag_tid=5, entry_tid=48022' 2026-02-19T20:32:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48022 2026-02-19T20:32:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 48022 2026-02-19T20:32:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:23.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:23.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:23.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:23.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:23.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:23.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:23.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:23.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:23.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:23.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:23.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=48324], [object_number=87, tag_tid=5, entry_tid=48323], [object_number=86, tag_tid=5, entry_tid=48322], [object_number=85, tag_tid=5, entry_tid=48321]]], state=connected] 2026-02-19T20:32:23.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=48324], [object_number=87, tag_tid=5, entry_tid=48323], [object_number=86, tag_tid=5, entry_tid=48322], [object_number=85, tag_tid=5, entry_tid=48321]]], state=connected] 2026-02-19T20:32:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=84, tag_tid=5, entry_tid=48324' 2026-02-19T20:32:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=84, tag_tid=5, entry_tid=48324' 2026-02-19T20:32:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=48324' '!=' 'object_number=86, tag_tid=5, entry_tid=48022' 2026-02-19T20:32:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=48324' = 'object_number=86, tag_tid=5, entry_tid=48022' 2026-02-19T20:32:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=84, tag_tid=5, entry_tid=48324' 2026-02-19T20:32:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=48324' 2026-02-19T20:32:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:23.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:23.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=48324' 2026-02-19T20:32:23.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:23.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48324 2026-02-19T20:32:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 48324 2026-02-19T20:32:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:23.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:23.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:23.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:23.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:23.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=48636], [object_number=87, tag_tid=5, entry_tid=48635], [object_number=86, tag_tid=5, entry_tid=48634], [object_number=85, tag_tid=5, entry_tid=48633]]], state=connected] 2026-02-19T20:32:23.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=48636], [object_number=87, tag_tid=5, entry_tid=48635], [object_number=86, tag_tid=5, entry_tid=48634], [object_number=85, tag_tid=5, entry_tid=48633]]], state=connected] 2026-02-19T20:32:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=84, tag_tid=5, entry_tid=48636' 2026-02-19T20:32:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=84, tag_tid=5, entry_tid=48636' 2026-02-19T20:32:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=48636' '!=' 'object_number=84, tag_tid=5, entry_tid=48324' 2026-02-19T20:32:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=48636' = 'object_number=84, tag_tid=5, entry_tid=48324' 2026-02-19T20:32:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=84, tag_tid=5, entry_tid=48636' 2026-02-19T20:32:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=48636' 2026-02-19T20:32:23.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:23.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=48636' 2026-02-19T20:32:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48636 2026-02-19T20:32:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 48636 2026-02-19T20:32:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:24.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:24.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:24.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:24.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:24.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=48976], [object_number=87, tag_tid=5, entry_tid=48975], [object_number=86, tag_tid=5, entry_tid=48974], [object_number=85, tag_tid=5, entry_tid=48973]]], state=connected] 2026-02-19T20:32:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:24.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:24.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=84, tag_tid=5, entry_tid=48976], [object_number=87, tag_tid=5, entry_tid=48975], [object_number=86, tag_tid=5, entry_tid=48974], [object_number=85, tag_tid=5, entry_tid=48973]]], state=connected] 2026-02-19T20:32:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=84, tag_tid=5, entry_tid=48976' 2026-02-19T20:32:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=84, tag_tid=5, entry_tid=48976' 2026-02-19T20:32:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=48976' '!=' 'object_number=84, tag_tid=5, entry_tid=48636' 2026-02-19T20:32:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=84, tag_tid=5, entry_tid=48976' = 'object_number=84, tag_tid=5, entry_tid=48636' 2026-02-19T20:32:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=84, tag_tid=5, entry_tid=48976' 2026-02-19T20:32:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=48976' 2026-02-19T20:32:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=84, tag_tid=5, entry_tid=48976' 2026-02-19T20:32:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:24.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48976 2026-02-19T20:32:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 48976 2026-02-19T20:32:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:24.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:24.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:24.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:24.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:24.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:24.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:24.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:24.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:24.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:24.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:24.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:24.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:24.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:24.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:24.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:24.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=49025], [object_number=84, tag_tid=5, entry_tid=49024], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:24.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:24.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:24.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:24.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:24.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:24.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:24.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:24.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=49025], [object_number=84, tag_tid=5, entry_tid=49024], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:24.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=49025' '!=' 'object_number=84, tag_tid=5, entry_tid=48976' 2026-02-19T20:32:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:32:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=49025' = 'object_number=84, tag_tid=5, entry_tid=48976' 2026-02-19T20:32:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:24.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=6 2026-02-19T20:32:24.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:24.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:32:24.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=5 2026-02-19T20:32:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:32:24.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:24.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:32:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:32:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=49025 2026-02-19T20:32:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 6 = 5 -a 1 -le 49025 2026-02-19T20:32:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:32:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:32:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:24.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:24.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:24.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:24.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=49025], [object_number=84, tag_tid=5, entry_tid=49024], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:25.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:25.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:25.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:25.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=49025], [object_number=84, tag_tid=5, entry_tid=49024], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=49025' '!=' 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:32:25.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:25.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:25.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:25.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:25.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:25.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:25.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:25.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:25.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:25.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:25.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:25.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:25.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:25.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:25.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:25.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:25.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:25.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:25.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=49025], [object_number=84, tag_tid=5, entry_tid=49024], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:25.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:25.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=49025], [object_number=84, tag_tid=5, entry_tid=49024], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=49025' '!=' 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:32:26.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:26.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:26.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:26.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:26.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:26.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:26.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:26.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:26.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:26.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:26.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:26.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:26.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:26.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:26.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:26.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:26.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:26.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=49025], [object_number=84, tag_tid=5, entry_tid=49024], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:26.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:26.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:26.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:26.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:26.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:26.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:26.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:26.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:26.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:26.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=49025], [object_number=84, tag_tid=5, entry_tid=49024], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:26.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=49025' '!=' 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:32:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:28.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:28.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:28.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:28.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:28.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:28.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:28.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:28.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:28.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:28.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:28.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:28.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=49025], [object_number=84, tag_tid=5, entry_tid=49024], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:28.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:28.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:28.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:28.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:28.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:28.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=5, entry_tid=49025], [object_number=84, tag_tid=5, entry_tid=49024], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:28.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:28.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:28.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:28.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=85, tag_tid=5, entry_tid=49025' '!=' 'object_number=85, tag_tid=5, entry_tid=49025' 2026-02-19T20:32:28.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:32:28.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:32:29.597 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3561872813928786) > weight (0.25) 2026-02-19T20:32:29.597 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-19T20:32:29.597 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-19T20:32:29.597 INFO:tasks.rbd_mirror_thrash:waiting for 1.17445538880103 secs before reviving daemons 2026-02-19T20:32:29.598 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T20:32:29.595+0000 7f8c227ff640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 19549) UID: 1000 2026-02-19T20:32:30.772 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-19T20:32:30.772 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:32:30.772 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-19T20:32:30.772 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-19T20:32:30.773 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-19T20:32:30.774 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-19T20:32:30.774 INFO:tasks.rbd_mirror_thrash:waiting for 68.2601576971575 secs before thrashing 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:32:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:32:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:32:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:32:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:32:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:32:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:32:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:32:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:32:30.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:32:30.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:32:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:32:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:32:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:32:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:32:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:32:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:32:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:32:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:32:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:32:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:32:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:32:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:32:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:32:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:32:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:32:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:32:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 21 2026-02-19T20:32:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 21 2026-02-19T20:32:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:32:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=85, tag_tid=6, entry_tid=1], [object_number=84, tag_tid=6, entry_tid=0], [object_number=87, tag_tid=5, entry_tid=49023], [object_number=86, tag_tid=5, entry_tid=49022]]], state=connected] 2026-02-19T20:32:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=85, tag_tid=6, entry_tid=1' -a 'object_number=85, tag_tid=6, entry_tid=1' = 'object_number=85, tag_tid=6, entry_tid=1' 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap2 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:32:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:32:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 3 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:32:22 2026 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:32:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:32:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n snap1-clean ']' 2026-02-19T20:32:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-19T20:32:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap1-clean 2026-02-19T20:32:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:32:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:32:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:32:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:32:32.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:32:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:32:32.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:32:33.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:32:33.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:32:33.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:32:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap2 2026-02-19T20:32:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap2 2026-02-19T20:32:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:32:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:32:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:32:33.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:32:33.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:32:33.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:32:33.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:32:34.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:32:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:32:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:32:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name=snap2-clean 2026-02-19T20:32:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap2-clean 2026-02-19T20:32:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:32:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap2-clean 2026-02-19T20:32:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2026-02-19T20:32:35.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:32:35.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror test 2026-02-19T20:32:35.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:32:35.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:35.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:35.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:32:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=10 2026-02-19T20:32:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:32:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-19T20:32:35.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 607232 bytes 2026-02-19T20:32:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 78020608 bytes 2026-02-19T20:32:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 128356352 bytes 2026-02-19T20:32:38.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 202760192 bytes 2026-02-19T20:32:38.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 236315648 bytes 2026-02-19T20:32:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 286601216 bytes 2026-02-19T20:32:40.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 336156672 bytes 2026-02-19T20:32:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 396113920 bytes 2026-02-19T20:32:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 518829056 bytes 2026-02-19T20:32:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 555524096 bytes 2026-02-19T20:32:43.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 626012160 bytes 2026-02-19T20:32:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 669130752 bytes 2026-02-19T20:32:44.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 673607680 bytes 2026-02-19T20:32:44.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 679964672 bytes 2026-02-19T20:32:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 695505920 bytes 2026-02-19T20:32:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 758159360 bytes 2026-02-19T20:32:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:32:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:32:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:32:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-19T20:32:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:32:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:32:45.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:32:45.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:32:45.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:32:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-19T20:32:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:32:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:32:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:32:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:32:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:32:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-19T20:32:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='test: 2026-02-19T20:32:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: global_id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:32:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+replaying 2026-02-19T20:32:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: replaying, {"bytes_per_second":12395228.4,"entries_behind_primary":4698,"entries_per_second":772.03,"non_primary_position":{"entry_tid":44328,"object_number":80,"tag_tid":5},"primary_position":{"entry_tid":1,"object_number":85,"tag_tid":6},"seconds_until_synced":6} 2026-02-19T20:32:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: service: mirror.1 on trial151 2026-02-19T20:32:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:32:17 2026-02-19T20:32:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: peer_sites: 2026-02-19T20:32:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: name: cluster2 2026-02-19T20:32:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+stopped 2026-02-19T20:32:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: local image is primary 2026-02-19T20:32:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:32:17' 2026-02-19T20:32:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ state: .*up+replaying' 2026-02-19T20:32:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: state: up+replaying 2026-02-19T20:32:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ description: .*' 2026-02-19T20:32:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: description: replaying, {"bytes_per_second":12395228.4,"entries_behind_primary":4698,"entries_per_second":772.03,"non_primary_position":{"entry_tid":44328,"object_number":80,"tag_tid":5},"primary_position":{"entry_tid":1,"object_number":85,"tag_tid":6},"seconds_until_synced":6} 2026-02-19T20:32:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:32:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:32:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^up+' 2026-02-19T20:32:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:32:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep 'service: *mirror..* on ' 2026-02-19T20:32:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: service: mirror.1 on trial151 2026-02-19T20:32:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update 2026-02-19T20:32:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-19T20:32:46.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:32:17' 2026-02-19T20:32:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-19 20:32:17' 2026-02-19T20:32:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:32:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:32:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'prev_last_update=2026-02-19 20:32:17' 2026-02-19T20:32:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:32:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:32:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-19T20:32:46.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='OKOKOK142064262mirror.1trial15120.3.0-5336-g802bf111falseOK42104271mirror.2trial15120.3.0-5336-g802bf111trueOK42144267mirror.3trial15120.3.0-5336-g802bf111falseOK43214324mirror.0trial15120.3.0-5336-g802bf111falseOKtest959deb13-de1a-48d9-a341-1f95416f8379up+replayingreplaying, {"bytes_per_second":12395228.4,"entries_behind_primary":4698,"entries_per_second":772.03,"non_primary_position":{"entry_tid":44328,"object_number":80,"tag_tid":5},"primary_position":{"entry_tid":1,"object_number":85,"tag_tid":6},"seconds_until_synced":6}42064262mirror.1trial1512026-02-19 20:32:17cluster2a054b997-3cfb-48e6-978a-70d6a85e56deup+stoppedlocal image is primary2026-02-19 20:32:17' 2026-02-19T20:32:46.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update state 2026-02-19T20:32:46.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-19T20:32:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:32:17' 2026-02-19T20:32:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-19T20:32:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=up+replaying 2026-02-19T20:32:46.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:32:46.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep up+replaying 2026-02-19T20:32:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:32:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:32:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap3 2026-02-19T20:32:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap3 2026-02-19T20:32:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:32:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:32:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:32:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap3 2026-02-19T20:32:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:33:18.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:33:18.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:18.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:18.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:18.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:33:18.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:18.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:18.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:18.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:33:18.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:33:18.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:18.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:33:19.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:33:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:33:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:33:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:33:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:33:19.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:33:19.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:33:19.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:33:19.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:19.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:33:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:33:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:33:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:33:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:33:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:33:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:33:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:33:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:33:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:33:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:33:19.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:19.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:33:19.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:19.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:19.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:33:19.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:33:19.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:33:19.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:19.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:33:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:33:19.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:33:19.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:33:19.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:33:19.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:33:19.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:33:19.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:33:19.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:33:19.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:33:19.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:20.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:20.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:20.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=115, tag_tid=8, entry_tid=27599], [object_number=114, tag_tid=8, entry_tid=27598], [object_number=113, tag_tid=8, entry_tid=27597], [object_number=112, tag_tid=8, entry_tid=27596]]], state=connected] 2026-02-19T20:33:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:20.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:20.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:20.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:20.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:20.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:20.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:20.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=115, tag_tid=8, entry_tid=27599], [object_number=114, tag_tid=8, entry_tid=27598], [object_number=113, tag_tid=8, entry_tid=27597], [object_number=112, tag_tid=8, entry_tid=27596]]], state=connected] 2026-02-19T20:33:20.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=115, tag_tid=8, entry_tid=27599' 2026-02-19T20:33:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=115, tag_tid=8, entry_tid=27599' 2026-02-19T20:33:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=115, tag_tid=8, entry_tid=27599' '!=' '' 2026-02-19T20:33:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=115, tag_tid=8, entry_tid=27599' = '' 2026-02-19T20:33:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=115, tag_tid=8, entry_tid=27599' 2026-02-19T20:33:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=115, tag_tid=8, entry_tid=27599' 2026-02-19T20:33:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=115, tag_tid=8, entry_tid=27599' 2026-02-19T20:33:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27599 2026-02-19T20:33:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 27599 2026-02-19T20:33:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:20.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:20.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:20.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:20.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:20.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=114, tag_tid=8, entry_tid=27922], [object_number=113, tag_tid=8, entry_tid=27921], [object_number=112, tag_tid=8, entry_tid=27920], [object_number=115, tag_tid=8, entry_tid=27919]]], state=connected] 2026-02-19T20:33:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:20.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:20.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:20.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:20.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:20.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:20.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:20.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=114, tag_tid=8, entry_tid=27922], [object_number=113, tag_tid=8, entry_tid=27921], [object_number=112, tag_tid=8, entry_tid=27920], [object_number=115, tag_tid=8, entry_tid=27919]]], state=connected] 2026-02-19T20:33:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=114, tag_tid=8, entry_tid=27922' 2026-02-19T20:33:20.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=114, tag_tid=8, entry_tid=27922' 2026-02-19T20:33:20.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=114, tag_tid=8, entry_tid=27922' '!=' 'object_number=115, tag_tid=8, entry_tid=27599' 2026-02-19T20:33:20.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:20.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=114, tag_tid=8, entry_tid=27922' = 'object_number=115, tag_tid=8, entry_tid=27599' 2026-02-19T20:33:20.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=114, tag_tid=8, entry_tid=27922' 2026-02-19T20:33:20.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:20.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:20.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=114, tag_tid=8, entry_tid=27922' 2026-02-19T20:33:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:20.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:20.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=114, tag_tid=8, entry_tid=27922' 2026-02-19T20:33:20.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27922 2026-02-19T20:33:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 27922 2026-02-19T20:33:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:20.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:20.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:20.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:20.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:20.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:20.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:20.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:20.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:20.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:20.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:20.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:20.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:20.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:20.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:20.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:20.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:20.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:20.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=113, tag_tid=8, entry_tid=28261], [object_number=112, tag_tid=8, entry_tid=28260], [object_number=115, tag_tid=8, entry_tid=28259], [object_number=114, tag_tid=8, entry_tid=28258]]], state=connected] 2026-02-19T20:33:20.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=113, tag_tid=8, entry_tid=28261], [object_number=112, tag_tid=8, entry_tid=28260], [object_number=115, tag_tid=8, entry_tid=28259], [object_number=114, tag_tid=8, entry_tid=28258]]], state=connected] 2026-02-19T20:33:21.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=113, tag_tid=8, entry_tid=28261' 2026-02-19T20:33:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=113, tag_tid=8, entry_tid=28261' 2026-02-19T20:33:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=113, tag_tid=8, entry_tid=28261' '!=' 'object_number=114, tag_tid=8, entry_tid=27922' 2026-02-19T20:33:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=113, tag_tid=8, entry_tid=28261' = 'object_number=114, tag_tid=8, entry_tid=27922' 2026-02-19T20:33:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=113, tag_tid=8, entry_tid=28261' 2026-02-19T20:33:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=113, tag_tid=8, entry_tid=28261' 2026-02-19T20:33:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:21.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=113, tag_tid=8, entry_tid=28261' 2026-02-19T20:33:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28261 2026-02-19T20:33:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 28261 2026-02-19T20:33:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:21.183 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T20:33:21.183 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T20:33:21.183 INFO:tasks.rbd_mirror_thrash:waiting for 6.5678023964094345 secs before reviving daemons 2026-02-19T20:33:21.184 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T20:33:21.178+0000 7fcde17fe640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 23893) UID: 1000 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:21.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:21.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:21.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:21.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=112, tag_tid=8, entry_tid=28552], [object_number=115, tag_tid=8, entry_tid=28551], [object_number=114, tag_tid=8, entry_tid=28550], [object_number=113, tag_tid=8, entry_tid=28549]]], state=connected] 2026-02-19T20:33:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=112, tag_tid=8, entry_tid=28552], [object_number=115, tag_tid=8, entry_tid=28551], [object_number=114, tag_tid=8, entry_tid=28550], [object_number=113, tag_tid=8, entry_tid=28549]]], state=connected] 2026-02-19T20:33:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=112, tag_tid=8, entry_tid=28552' 2026-02-19T20:33:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=112, tag_tid=8, entry_tid=28552' 2026-02-19T20:33:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=112, tag_tid=8, entry_tid=28552' '!=' 'object_number=113, tag_tid=8, entry_tid=28261' 2026-02-19T20:33:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=112, tag_tid=8, entry_tid=28552' = 'object_number=113, tag_tid=8, entry_tid=28261' 2026-02-19T20:33:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=112, tag_tid=8, entry_tid=28552' 2026-02-19T20:33:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=112, tag_tid=8, entry_tid=28552' 2026-02-19T20:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:21.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:21.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:21.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:21.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=112, tag_tid=8, entry_tid=28552' 2026-02-19T20:33:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28552 2026-02-19T20:33:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 28552 2026-02-19T20:33:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:21.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:21.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:21.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:21.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:21.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:21.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:21.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:22.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:22.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:22.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:22.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=114, tag_tid=8, entry_tid=28982], [object_number=113, tag_tid=8, entry_tid=28981], [object_number=112, tag_tid=8, entry_tid=28980], [object_number=115, tag_tid=8, entry_tid=28979]]], state=connected] 2026-02-19T20:33:22.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:22.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:22.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:22.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:22.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:22.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:22.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=114, tag_tid=8, entry_tid=28982], [object_number=113, tag_tid=8, entry_tid=28981], [object_number=112, tag_tid=8, entry_tid=28980], [object_number=115, tag_tid=8, entry_tid=28979]]], state=connected] 2026-02-19T20:33:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=114, tag_tid=8, entry_tid=28982' 2026-02-19T20:33:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=114, tag_tid=8, entry_tid=28982' 2026-02-19T20:33:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=114, tag_tid=8, entry_tid=28982' '!=' 'object_number=112, tag_tid=8, entry_tid=28552' 2026-02-19T20:33:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=114, tag_tid=8, entry_tid=28982' = 'object_number=112, tag_tid=8, entry_tid=28552' 2026-02-19T20:33:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=114, tag_tid=8, entry_tid=28982' 2026-02-19T20:33:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=114, tag_tid=8, entry_tid=28982' 2026-02-19T20:33:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:22.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:22.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:22.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:22.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:22.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:22.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=114, tag_tid=8, entry_tid=28982' 2026-02-19T20:33:22.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:22.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:22.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28982 2026-02-19T20:33:22.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 28982 2026-02-19T20:33:22.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:22.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:22.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:22.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:22.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:22.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:22.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:22.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:22.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:22.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:22.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=115, tag_tid=8, entry_tid=29339], [object_number=114, tag_tid=8, entry_tid=29338], [object_number=113, tag_tid=8, entry_tid=29337], [object_number=112, tag_tid=8, entry_tid=29336]]], state=connected] 2026-02-19T20:33:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=115, tag_tid=8, entry_tid=29339], [object_number=114, tag_tid=8, entry_tid=29338], [object_number=113, tag_tid=8, entry_tid=29337], [object_number=112, tag_tid=8, entry_tid=29336]]], state=connected] 2026-02-19T20:33:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=115, tag_tid=8, entry_tid=29339' 2026-02-19T20:33:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=115, tag_tid=8, entry_tid=29339' 2026-02-19T20:33:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=115, tag_tid=8, entry_tid=29339' '!=' 'object_number=114, tag_tid=8, entry_tid=28982' 2026-02-19T20:33:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=115, tag_tid=8, entry_tid=29339' = 'object_number=114, tag_tid=8, entry_tid=28982' 2026-02-19T20:33:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=115, tag_tid=8, entry_tid=29339' 2026-02-19T20:33:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=115, tag_tid=8, entry_tid=29339' 2026-02-19T20:33:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:22.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=115, tag_tid=8, entry_tid=29339' 2026-02-19T20:33:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29339 2026-02-19T20:33:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 29339 2026-02-19T20:33:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:22.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=114, tag_tid=8, entry_tid=29830], [object_number=113, tag_tid=8, entry_tid=29829], [object_number=112, tag_tid=8, entry_tid=29828], [object_number=115, tag_tid=8, entry_tid=29827]]], state=connected] 2026-02-19T20:33:23.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=114, tag_tid=8, entry_tid=29830], [object_number=113, tag_tid=8, entry_tid=29829], [object_number=112, tag_tid=8, entry_tid=29828], [object_number=115, tag_tid=8, entry_tid=29827]]], state=connected] 2026-02-19T20:33:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=114, tag_tid=8, entry_tid=29830' 2026-02-19T20:33:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=114, tag_tid=8, entry_tid=29830' 2026-02-19T20:33:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=114, tag_tid=8, entry_tid=29830' '!=' 'object_number=115, tag_tid=8, entry_tid=29339' 2026-02-19T20:33:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=114, tag_tid=8, entry_tid=29830' = 'object_number=115, tag_tid=8, entry_tid=29339' 2026-02-19T20:33:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=114, tag_tid=8, entry_tid=29830' 2026-02-19T20:33:23.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:23.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:23.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:23.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=114, tag_tid=8, entry_tid=29830' 2026-02-19T20:33:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=114, tag_tid=8, entry_tid=29830' 2026-02-19T20:33:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29830 2026-02-19T20:33:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 29830 2026-02-19T20:33:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:23.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:23.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:23.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:23.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:23.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:23.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:23.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=112, tag_tid=8, entry_tid=30164], [object_number=115, tag_tid=8, entry_tid=30163], [object_number=114, tag_tid=8, entry_tid=30162], [object_number=113, tag_tid=8, entry_tid=30161]]], state=connected] 2026-02-19T20:33:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:23.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:23.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:23.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:23.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:23.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:23.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:23.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:23.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=112, tag_tid=8, entry_tid=30164], [object_number=115, tag_tid=8, entry_tid=30163], [object_number=114, tag_tid=8, entry_tid=30162], [object_number=113, tag_tid=8, entry_tid=30161]]], state=connected] 2026-02-19T20:33:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=112, tag_tid=8, entry_tid=30164' 2026-02-19T20:33:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=112, tag_tid=8, entry_tid=30164' 2026-02-19T20:33:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=112, tag_tid=8, entry_tid=30164' '!=' 'object_number=114, tag_tid=8, entry_tid=29830' 2026-02-19T20:33:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=112, tag_tid=8, entry_tid=30164' = 'object_number=114, tag_tid=8, entry_tid=29830' 2026-02-19T20:33:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=112, tag_tid=8, entry_tid=30164' 2026-02-19T20:33:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:23.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:23.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:23.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:23.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=112, tag_tid=8, entry_tid=30164' 2026-02-19T20:33:23.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:23.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:23.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:23.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:23.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=112, tag_tid=8, entry_tid=30164' 2026-02-19T20:33:23.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:23.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30164 2026-02-19T20:33:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 30164 2026-02-19T20:33:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:23.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:23.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:23.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:23.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:23.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=113, tag_tid=8, entry_tid=30449], [object_number=112, tag_tid=8, entry_tid=30448], [object_number=115, tag_tid=8, entry_tid=30447], [object_number=114, tag_tid=8, entry_tid=30446]]], state=connected] 2026-02-19T20:33:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 28 2026-02-19T20:33:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=113, tag_tid=8, entry_tid=30449], [object_number=112, tag_tid=8, entry_tid=30448], [object_number=115, tag_tid=8, entry_tid=30447], [object_number=114, tag_tid=8, entry_tid=30446]]], state=connected] 2026-02-19T20:33:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=113, tag_tid=8, entry_tid=30449' 2026-02-19T20:33:24.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=113, tag_tid=8, entry_tid=30449' 2026-02-19T20:33:24.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=113, tag_tid=8, entry_tid=30449' '!=' 'object_number=112, tag_tid=8, entry_tid=30164' 2026-02-19T20:33:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=113, tag_tid=8, entry_tid=30449' = 'object_number=112, tag_tid=8, entry_tid=30164' 2026-02-19T20:33:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=113, tag_tid=8, entry_tid=30449' 2026-02-19T20:33:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=113, tag_tid=8, entry_tid=30449' 2026-02-19T20:33:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=113, tag_tid=8, entry_tid=30449' 2026-02-19T20:33:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30449 2026-02-19T20:33:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 30449 2026-02-19T20:33:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:24.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:24.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:24.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:24.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:24.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:24.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:24.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:24.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:24.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:24.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=30665], [object_number=116, tag_tid=8, entry_tid=30664], [object_number=119, tag_tid=8, entry_tid=30663], [object_number=118, tag_tid=8, entry_tid=30662]]], state=connected] 2026-02-19T20:33:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:24.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:24.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:24.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:24.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:24.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=30665], [object_number=116, tag_tid=8, entry_tid=30664], [object_number=119, tag_tid=8, entry_tid=30663], [object_number=118, tag_tid=8, entry_tid=30662]]], state=connected] 2026-02-19T20:33:24.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=117, tag_tid=8, entry_tid=30665' 2026-02-19T20:33:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=117, tag_tid=8, entry_tid=30665' 2026-02-19T20:33:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=30665' '!=' 'object_number=113, tag_tid=8, entry_tid=30449' 2026-02-19T20:33:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=30665' = 'object_number=113, tag_tid=8, entry_tid=30449' 2026-02-19T20:33:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=117, tag_tid=8, entry_tid=30665' 2026-02-19T20:33:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=30665' 2026-02-19T20:33:24.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:24.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:24.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:24.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:24.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=30665' 2026-02-19T20:33:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30665 2026-02-19T20:33:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 30665 2026-02-19T20:33:24.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:24.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:24.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:24.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:24.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:24.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:24.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:24.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:24.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:24.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:24.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:24.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:24.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:24.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:24.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=31041], [object_number=116, tag_tid=8, entry_tid=31040], [object_number=119, tag_tid=8, entry_tid=31039], [object_number=118, tag_tid=8, entry_tid=31038]]], state=connected] 2026-02-19T20:33:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:24.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:24.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:24.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:24.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=31041], [object_number=116, tag_tid=8, entry_tid=31040], [object_number=119, tag_tid=8, entry_tid=31039], [object_number=118, tag_tid=8, entry_tid=31038]]], state=connected] 2026-02-19T20:33:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:24.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=117, tag_tid=8, entry_tid=31041' 2026-02-19T20:33:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=117, tag_tid=8, entry_tid=31041' 2026-02-19T20:33:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=31041' '!=' 'object_number=117, tag_tid=8, entry_tid=30665' 2026-02-19T20:33:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=31041' = 'object_number=117, tag_tid=8, entry_tid=30665' 2026-02-19T20:33:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=117, tag_tid=8, entry_tid=31041' 2026-02-19T20:33:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=31041' 2026-02-19T20:33:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=31041' 2026-02-19T20:33:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31041 2026-02-19T20:33:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 31041 2026-02-19T20:33:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:25.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:25.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=31346], [object_number=117, tag_tid=8, entry_tid=31345], [object_number=116, tag_tid=8, entry_tid=31344], [object_number=119, tag_tid=8, entry_tid=31343]]], state=connected] 2026-02-19T20:33:25.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:25.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:25.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:25.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:25.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:25.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:25.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=31346], [object_number=117, tag_tid=8, entry_tid=31345], [object_number=116, tag_tid=8, entry_tid=31344], [object_number=119, tag_tid=8, entry_tid=31343]]], state=connected] 2026-02-19T20:33:25.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=118, tag_tid=8, entry_tid=31346' 2026-02-19T20:33:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=118, tag_tid=8, entry_tid=31346' 2026-02-19T20:33:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=31346' '!=' 'object_number=117, tag_tid=8, entry_tid=31041' 2026-02-19T20:33:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:25.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=31346' = 'object_number=117, tag_tid=8, entry_tid=31041' 2026-02-19T20:33:25.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=118, tag_tid=8, entry_tid=31346' 2026-02-19T20:33:25.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:25.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:25.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:25.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:25.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=31346' 2026-02-19T20:33:25.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:25.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:25.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=31346' 2026-02-19T20:33:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:25.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31346 2026-02-19T20:33:25.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 31346 2026-02-19T20:33:25.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:25.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:25.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:25.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:25.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:25.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:25.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:25.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:25.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:25.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:25.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:25.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:25.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:25.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=31632], [object_number=119, tag_tid=8, entry_tid=31631], [object_number=118, tag_tid=8, entry_tid=31630], [object_number=117, tag_tid=8, entry_tid=31629]]], state=connected] 2026-02-19T20:33:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:25.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:25.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:25.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=31632], [object_number=119, tag_tid=8, entry_tid=31631], [object_number=118, tag_tid=8, entry_tid=31630], [object_number=117, tag_tid=8, entry_tid=31629]]], state=connected] 2026-02-19T20:33:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=116, tag_tid=8, entry_tid=31632' 2026-02-19T20:33:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=116, tag_tid=8, entry_tid=31632' 2026-02-19T20:33:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=31632' '!=' 'object_number=118, tag_tid=8, entry_tid=31346' 2026-02-19T20:33:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=31632' = 'object_number=118, tag_tid=8, entry_tid=31346' 2026-02-19T20:33:25.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=116, tag_tid=8, entry_tid=31632' 2026-02-19T20:33:25.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:25.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:25.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:25.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:25.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=31632' 2026-02-19T20:33:25.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:25.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:25.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:25.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:25.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:25.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:25.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=31632' 2026-02-19T20:33:25.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:25.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:25.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31632 2026-02-19T20:33:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 31632 2026-02-19T20:33:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:25.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:25.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:25.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:25.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=31944], [object_number=119, tag_tid=8, entry_tid=31943], [object_number=118, tag_tid=8, entry_tid=31942], [object_number=117, tag_tid=8, entry_tid=31941]]], state=connected] 2026-02-19T20:33:26.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=31944], [object_number=119, tag_tid=8, entry_tid=31943], [object_number=118, tag_tid=8, entry_tid=31942], [object_number=117, tag_tid=8, entry_tid=31941]]], state=connected] 2026-02-19T20:33:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=116, tag_tid=8, entry_tid=31944' 2026-02-19T20:33:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=116, tag_tid=8, entry_tid=31944' 2026-02-19T20:33:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=31944' '!=' 'object_number=116, tag_tid=8, entry_tid=31632' 2026-02-19T20:33:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=31944' = 'object_number=116, tag_tid=8, entry_tid=31632' 2026-02-19T20:33:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=116, tag_tid=8, entry_tid=31944' 2026-02-19T20:33:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=31944' 2026-02-19T20:33:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:26.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:26.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:26.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=31944' 2026-02-19T20:33:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31944 2026-02-19T20:33:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 31944 2026-02-19T20:33:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:26.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:26.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:26.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:26.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:26.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:26.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:26.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:26.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:26.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:26.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:26.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:26.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:26.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:26.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:26.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:26.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:26.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:26.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:26.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:26.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:26.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:26.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:26.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:26.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=32254], [object_number=117, tag_tid=8, entry_tid=32253], [object_number=116, tag_tid=8, entry_tid=32252], [object_number=119, tag_tid=8, entry_tid=32251]]], state=connected] 2026-02-19T20:33:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:26.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:26.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:26.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:26.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:26.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:26.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:26.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:26.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:26.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:26.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:26.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=32254], [object_number=117, tag_tid=8, entry_tid=32253], [object_number=116, tag_tid=8, entry_tid=32252], [object_number=119, tag_tid=8, entry_tid=32251]]], state=connected] 2026-02-19T20:33:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=118, tag_tid=8, entry_tid=32254' 2026-02-19T20:33:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=118, tag_tid=8, entry_tid=32254' 2026-02-19T20:33:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=32254' '!=' 'object_number=116, tag_tid=8, entry_tid=31944' 2026-02-19T20:33:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=32254' = 'object_number=116, tag_tid=8, entry_tid=31944' 2026-02-19T20:33:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=118, tag_tid=8, entry_tid=32254' 2026-02-19T20:33:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:26.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=32254' 2026-02-19T20:33:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:26.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=32254' 2026-02-19T20:33:26.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:26.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32254 2026-02-19T20:33:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 32254 2026-02-19T20:33:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:26.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:26.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:26.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:26.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:26.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:26.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=119, tag_tid=8, entry_tid=32543], [object_number=118, tag_tid=8, entry_tid=32542], [object_number=117, tag_tid=8, entry_tid=32541], [object_number=116, tag_tid=8, entry_tid=32540]]], state=connected] 2026-02-19T20:33:26.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=119, tag_tid=8, entry_tid=32543], [object_number=118, tag_tid=8, entry_tid=32542], [object_number=117, tag_tid=8, entry_tid=32541], [object_number=116, tag_tid=8, entry_tid=32540]]], state=connected] 2026-02-19T20:33:26.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=119, tag_tid=8, entry_tid=32543' 2026-02-19T20:33:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=119, tag_tid=8, entry_tid=32543' 2026-02-19T20:33:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=119, tag_tid=8, entry_tid=32543' '!=' 'object_number=118, tag_tid=8, entry_tid=32254' 2026-02-19T20:33:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=119, tag_tid=8, entry_tid=32543' = 'object_number=118, tag_tid=8, entry_tid=32254' 2026-02-19T20:33:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=119, tag_tid=8, entry_tid=32543' 2026-02-19T20:33:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=119, tag_tid=8, entry_tid=32543' 2026-02-19T20:33:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=119, tag_tid=8, entry_tid=32543' 2026-02-19T20:33:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:26.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32543 2026-02-19T20:33:26.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 32543 2026-02-19T20:33:26.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:26.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:26.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:27.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:27.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:27.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:27.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:27.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:27.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:27.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:27.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:27.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=32861], [object_number=116, tag_tid=8, entry_tid=32860], [object_number=119, tag_tid=8, entry_tid=32859], [object_number=118, tag_tid=8, entry_tid=32858]]], state=connected] 2026-02-19T20:33:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:27.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:27.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:27.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:27.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:27.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:27.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:27.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:27.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=32861], [object_number=116, tag_tid=8, entry_tid=32860], [object_number=119, tag_tid=8, entry_tid=32859], [object_number=118, tag_tid=8, entry_tid=32858]]], state=connected] 2026-02-19T20:33:27.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=117, tag_tid=8, entry_tid=32861' 2026-02-19T20:33:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=117, tag_tid=8, entry_tid=32861' 2026-02-19T20:33:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=32861' '!=' 'object_number=119, tag_tid=8, entry_tid=32543' 2026-02-19T20:33:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=32861' = 'object_number=119, tag_tid=8, entry_tid=32543' 2026-02-19T20:33:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=117, tag_tid=8, entry_tid=32861' 2026-02-19T20:33:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=32861' 2026-02-19T20:33:27.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:27.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:27.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=32861' 2026-02-19T20:33:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32861 2026-02-19T20:33:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 32861 2026-02-19T20:33:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:27.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:27.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:27.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:27.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:27.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:27.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:27.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:33:27.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:27.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:27.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:27.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:27.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:27.752 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T20:33:27.752 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:33:27.752 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T20:33:27.752 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T20:33:27.753 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:27.754 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T20:33:27.754 INFO:tasks.rbd_mirror_thrash:waiting for 68.5227917063123 secs before thrashing 2026-02-19T20:33:27.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:27.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:27.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:27.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:27.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:27.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=33158], [object_number=117, tag_tid=8, entry_tid=33157], [object_number=116, tag_tid=8, entry_tid=33156], [object_number=119, tag_tid=8, entry_tid=33155]]], state=connected] 2026-02-19T20:33:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=33158], [object_number=117, tag_tid=8, entry_tid=33157], [object_number=116, tag_tid=8, entry_tid=33156], [object_number=119, tag_tid=8, entry_tid=33155]]], state=connected] 2026-02-19T20:33:27.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:27.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=118, tag_tid=8, entry_tid=33158' 2026-02-19T20:33:27.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=118, tag_tid=8, entry_tid=33158' 2026-02-19T20:33:27.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=33158' '!=' 'object_number=117, tag_tid=8, entry_tid=32861' 2026-02-19T20:33:27.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=33158' = 'object_number=117, tag_tid=8, entry_tid=32861' 2026-02-19T20:33:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=118, tag_tid=8, entry_tid=33158' 2026-02-19T20:33:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:27.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:27.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=33158' 2026-02-19T20:33:27.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:27.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:27.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:27.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:27.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=33158' 2026-02-19T20:33:27.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:27.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:27.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33158 2026-02-19T20:33:27.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 33158 2026-02-19T20:33:27.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:27.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:27.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:28.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:28.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:28.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:28.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:28.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:28.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:28.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:28.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:28.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:28.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:28.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:28.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:28.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:28.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:28.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:28.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:28.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:28.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:28.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=33437], [object_number=116, tag_tid=8, entry_tid=33436], [object_number=119, tag_tid=8, entry_tid=33435], [object_number=118, tag_tid=8, entry_tid=33434]]], state=connected] 2026-02-19T20:33:28.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:28.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:28.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:28.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=33437], [object_number=116, tag_tid=8, entry_tid=33436], [object_number=119, tag_tid=8, entry_tid=33435], [object_number=118, tag_tid=8, entry_tid=33434]]], state=connected] 2026-02-19T20:33:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=117, tag_tid=8, entry_tid=33437' 2026-02-19T20:33:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=117, tag_tid=8, entry_tid=33437' 2026-02-19T20:33:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=33437' '!=' 'object_number=118, tag_tid=8, entry_tid=33158' 2026-02-19T20:33:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=33437' = 'object_number=118, tag_tid=8, entry_tid=33158' 2026-02-19T20:33:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=117, tag_tid=8, entry_tid=33437' 2026-02-19T20:33:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=33437' 2026-02-19T20:33:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:28.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:28.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:28.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=33437' 2026-02-19T20:33:28.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:28.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33437 2026-02-19T20:33:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 33437 2026-02-19T20:33:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:28.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:28.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:28.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:28.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:28.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:28.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:28.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:28.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:28.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:28.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:28.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:28.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:28.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:28.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:28.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=33757], [object_number=116, tag_tid=8, entry_tid=33756], [object_number=119, tag_tid=8, entry_tid=33755], [object_number=118, tag_tid=8, entry_tid=33754]]], state=connected] 2026-02-19T20:33:28.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:28.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:28.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:28.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=33757], [object_number=116, tag_tid=8, entry_tid=33756], [object_number=119, tag_tid=8, entry_tid=33755], [object_number=118, tag_tid=8, entry_tid=33754]]], state=connected] 2026-02-19T20:33:28.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:28.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=117, tag_tid=8, entry_tid=33757' 2026-02-19T20:33:28.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=117, tag_tid=8, entry_tid=33757' 2026-02-19T20:33:28.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=33757' '!=' 'object_number=117, tag_tid=8, entry_tid=33437' 2026-02-19T20:33:28.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:28.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=33757' = 'object_number=117, tag_tid=8, entry_tid=33437' 2026-02-19T20:33:28.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=117, tag_tid=8, entry_tid=33757' 2026-02-19T20:33:28.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:28.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:28.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:28.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:28.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=33757' 2026-02-19T20:33:28.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:28.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:28.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:28.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:28.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:28.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:28.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=33757' 2026-02-19T20:33:28.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:28.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33757 2026-02-19T20:33:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 33757 2026-02-19T20:33:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:28.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:28.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:28.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:28.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=34050], [object_number=117, tag_tid=8, entry_tid=34049], [object_number=116, tag_tid=8, entry_tid=34048], [object_number=119, tag_tid=8, entry_tid=34047]]], state=connected] 2026-02-19T20:33:29.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:29.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:29.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:29.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:29.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:29.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:29.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:29.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=34050], [object_number=117, tag_tid=8, entry_tid=34049], [object_number=116, tag_tid=8, entry_tid=34048], [object_number=119, tag_tid=8, entry_tid=34047]]], state=connected] 2026-02-19T20:33:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=118, tag_tid=8, entry_tid=34050' 2026-02-19T20:33:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=118, tag_tid=8, entry_tid=34050' 2026-02-19T20:33:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=34050' '!=' 'object_number=117, tag_tid=8, entry_tid=33757' 2026-02-19T20:33:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=34050' = 'object_number=117, tag_tid=8, entry_tid=33757' 2026-02-19T20:33:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=118, tag_tid=8, entry_tid=34050' 2026-02-19T20:33:29.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:29.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:29.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=34050' 2026-02-19T20:33:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=34050' 2026-02-19T20:33:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34050 2026-02-19T20:33:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 34050 2026-02-19T20:33:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:29.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:29.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:29.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:29.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:29.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:29.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:29.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:29.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:29.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:29.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:29.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:29.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=34392], [object_number=119, tag_tid=8, entry_tid=34391], [object_number=118, tag_tid=8, entry_tid=34390], [object_number=117, tag_tid=8, entry_tid=34389]]], state=connected] 2026-02-19T20:33:29.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:29.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:29.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:29.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:29.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:29.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:29.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:29.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:29.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:29.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=34392], [object_number=119, tag_tid=8, entry_tid=34391], [object_number=118, tag_tid=8, entry_tid=34390], [object_number=117, tag_tid=8, entry_tid=34389]]], state=connected] 2026-02-19T20:33:29.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=116, tag_tid=8, entry_tid=34392' 2026-02-19T20:33:29.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=116, tag_tid=8, entry_tid=34392' 2026-02-19T20:33:29.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=34392' '!=' 'object_number=118, tag_tid=8, entry_tid=34050' 2026-02-19T20:33:29.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:29.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=34392' = 'object_number=118, tag_tid=8, entry_tid=34050' 2026-02-19T20:33:29.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=116, tag_tid=8, entry_tid=34392' 2026-02-19T20:33:29.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:29.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:29.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:29.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=34392' 2026-02-19T20:33:29.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:29.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:29.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:29.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:29.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=34392' 2026-02-19T20:33:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34392 2026-02-19T20:33:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 34392 2026-02-19T20:33:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:29.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:29.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:29.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:29.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:29.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:29.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:29.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:29.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:29.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:29.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:29.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:29.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:29.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:29.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:29.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:29.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:29.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:29.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:29.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:29.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=34704], [object_number=119, tag_tid=8, entry_tid=34703], [object_number=118, tag_tid=8, entry_tid=34702], [object_number=117, tag_tid=8, entry_tid=34701]]], state=connected] 2026-02-19T20:33:29.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:29.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:29.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 29 2026-02-19T20:33:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=34704], [object_number=119, tag_tid=8, entry_tid=34703], [object_number=118, tag_tid=8, entry_tid=34702], [object_number=117, tag_tid=8, entry_tid=34701]]], state=connected] 2026-02-19T20:33:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=116, tag_tid=8, entry_tid=34704' 2026-02-19T20:33:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=116, tag_tid=8, entry_tid=34704' 2026-02-19T20:33:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=34704' '!=' 'object_number=116, tag_tid=8, entry_tid=34392' 2026-02-19T20:33:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=34704' = 'object_number=116, tag_tid=8, entry_tid=34392' 2026-02-19T20:33:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=116, tag_tid=8, entry_tid=34704' 2026-02-19T20:33:29.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:29.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:29.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:29.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=34704' 2026-02-19T20:33:29.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:29.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:29.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=34704' 2026-02-19T20:33:29.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:29.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:29.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34704 2026-02-19T20:33:29.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 34704 2026-02-19T20:33:29.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:29.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:29.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:30.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:30.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:30.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:30.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=34920], [object_number=123, tag_tid=8, entry_tid=34919], [object_number=122, tag_tid=8, entry_tid=34918], [object_number=121, tag_tid=8, entry_tid=34917]]], state=connected] 2026-02-19T20:33:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=34920], [object_number=123, tag_tid=8, entry_tid=34919], [object_number=122, tag_tid=8, entry_tid=34918], [object_number=121, tag_tid=8, entry_tid=34917]]], state=connected] 2026-02-19T20:33:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=120, tag_tid=8, entry_tid=34920' 2026-02-19T20:33:30.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=120, tag_tid=8, entry_tid=34920' 2026-02-19T20:33:30.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=34920' '!=' 'object_number=116, tag_tid=8, entry_tid=34704' 2026-02-19T20:33:30.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:30.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=34920' = 'object_number=116, tag_tid=8, entry_tid=34704' 2026-02-19T20:33:30.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=120, tag_tid=8, entry_tid=34920' 2026-02-19T20:33:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=34920' 2026-02-19T20:33:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:30.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:30.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:30.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:30.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=34920' 2026-02-19T20:33:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34920 2026-02-19T20:33:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 34920 2026-02-19T20:33:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:30.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:30.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:30.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:30.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:30.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:30.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:30.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:30.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:30.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:30.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:30.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:30.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:30.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:30.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:30.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:30.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=35275], [object_number=122, tag_tid=8, entry_tid=35274], [object_number=121, tag_tid=8, entry_tid=35273], [object_number=120, tag_tid=8, entry_tid=35272]]], state=connected] 2026-02-19T20:33:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=35275], [object_number=122, tag_tid=8, entry_tid=35274], [object_number=121, tag_tid=8, entry_tid=35273], [object_number=120, tag_tid=8, entry_tid=35272]]], state=connected] 2026-02-19T20:33:30.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=123, tag_tid=8, entry_tid=35275' 2026-02-19T20:33:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=123, tag_tid=8, entry_tid=35275' 2026-02-19T20:33:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=35275' '!=' 'object_number=120, tag_tid=8, entry_tid=34920' 2026-02-19T20:33:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=35275' = 'object_number=120, tag_tid=8, entry_tid=34920' 2026-02-19T20:33:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=123, tag_tid=8, entry_tid=35275' 2026-02-19T20:33:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=35275' 2026-02-19T20:33:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=35275' 2026-02-19T20:33:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35275 2026-02-19T20:33:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 35275 2026-02-19T20:33:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:30.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:30.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:30.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:30.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:30.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:30.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:31.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:31.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:31.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:31.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:31.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:31.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:31.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:31.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:31.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:31.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:31.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:31.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:31.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:31.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:31.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:31.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:31.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:31.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:31.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:31.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:31.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=122, tag_tid=8, entry_tid=35582], [object_number=121, tag_tid=8, entry_tid=35581], [object_number=120, tag_tid=8, entry_tid=35580], [object_number=123, tag_tid=8, entry_tid=35579]]], state=connected] 2026-02-19T20:33:31.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:31.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=122, tag_tid=8, entry_tid=35582], [object_number=121, tag_tid=8, entry_tid=35581], [object_number=120, tag_tid=8, entry_tid=35580], [object_number=123, tag_tid=8, entry_tid=35579]]], state=connected] 2026-02-19T20:33:31.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=122, tag_tid=8, entry_tid=35582' 2026-02-19T20:33:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=122, tag_tid=8, entry_tid=35582' 2026-02-19T20:33:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=122, tag_tid=8, entry_tid=35582' '!=' 'object_number=123, tag_tid=8, entry_tid=35275' 2026-02-19T20:33:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=122, tag_tid=8, entry_tid=35582' = 'object_number=123, tag_tid=8, entry_tid=35275' 2026-02-19T20:33:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=122, tag_tid=8, entry_tid=35582' 2026-02-19T20:33:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:31.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=122, tag_tid=8, entry_tid=35582' 2026-02-19T20:33:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:31.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:31.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:31.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:31.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:31.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=122, tag_tid=8, entry_tid=35582' 2026-02-19T20:33:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35582 2026-02-19T20:33:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 35582 2026-02-19T20:33:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:31.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:31.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:31.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:31.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:31.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:31.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:31.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=122, tag_tid=8, entry_tid=35878], [object_number=121, tag_tid=8, entry_tid=35877], [object_number=120, tag_tid=8, entry_tid=35876], [object_number=123, tag_tid=8, entry_tid=35875]]], state=connected] 2026-02-19T20:33:31.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:31.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:31.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:31.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:31.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:31.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:31.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:31.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=122, tag_tid=8, entry_tid=35878], [object_number=121, tag_tid=8, entry_tid=35877], [object_number=120, tag_tid=8, entry_tid=35876], [object_number=123, tag_tid=8, entry_tid=35875]]], state=connected] 2026-02-19T20:33:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=122, tag_tid=8, entry_tid=35878' 2026-02-19T20:33:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=122, tag_tid=8, entry_tid=35878' 2026-02-19T20:33:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=122, tag_tid=8, entry_tid=35878' '!=' 'object_number=122, tag_tid=8, entry_tid=35582' 2026-02-19T20:33:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=122, tag_tid=8, entry_tid=35878' = 'object_number=122, tag_tid=8, entry_tid=35582' 2026-02-19T20:33:31.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=122, tag_tid=8, entry_tid=35878' 2026-02-19T20:33:31.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:31.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:31.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=122, tag_tid=8, entry_tid=35878' 2026-02-19T20:33:31.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:31.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=122, tag_tid=8, entry_tid=35878' 2026-02-19T20:33:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35878 2026-02-19T20:33:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 35878 2026-02-19T20:33:31.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:31.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:31.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:32.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=36176], [object_number=123, tag_tid=8, entry_tid=36175], [object_number=122, tag_tid=8, entry_tid=36174], [object_number=121, tag_tid=8, entry_tid=36173]]], state=connected] 2026-02-19T20:33:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:32.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:32.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=36176], [object_number=123, tag_tid=8, entry_tid=36175], [object_number=122, tag_tid=8, entry_tid=36174], [object_number=121, tag_tid=8, entry_tid=36173]]], state=connected] 2026-02-19T20:33:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=120, tag_tid=8, entry_tid=36176' 2026-02-19T20:33:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=120, tag_tid=8, entry_tid=36176' 2026-02-19T20:33:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=36176' '!=' 'object_number=122, tag_tid=8, entry_tid=35878' 2026-02-19T20:33:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=36176' = 'object_number=122, tag_tid=8, entry_tid=35878' 2026-02-19T20:33:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=120, tag_tid=8, entry_tid=36176' 2026-02-19T20:33:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:32.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=36176' 2026-02-19T20:33:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=36176' 2026-02-19T20:33:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36176 2026-02-19T20:33:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 36176 2026-02-19T20:33:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:32.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:32.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:32.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:32.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:32.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:32.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:32.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:32.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=122, tag_tid=8, entry_tid=36486], [object_number=121, tag_tid=8, entry_tid=36485], [object_number=120, tag_tid=8, entry_tid=36484], [object_number=123, tag_tid=8, entry_tid=36483]]], state=connected] 2026-02-19T20:33:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:32.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:32.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:32.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=122, tag_tid=8, entry_tid=36486], [object_number=121, tag_tid=8, entry_tid=36485], [object_number=120, tag_tid=8, entry_tid=36484], [object_number=123, tag_tid=8, entry_tid=36483]]], state=connected] 2026-02-19T20:33:32.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=122, tag_tid=8, entry_tid=36486' 2026-02-19T20:33:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=122, tag_tid=8, entry_tid=36486' 2026-02-19T20:33:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=122, tag_tid=8, entry_tid=36486' '!=' 'object_number=120, tag_tid=8, entry_tid=36176' 2026-02-19T20:33:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=122, tag_tid=8, entry_tid=36486' = 'object_number=120, tag_tid=8, entry_tid=36176' 2026-02-19T20:33:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=122, tag_tid=8, entry_tid=36486' 2026-02-19T20:33:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=122, tag_tid=8, entry_tid=36486' 2026-02-19T20:33:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:32.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:32.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:32.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:32.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=122, tag_tid=8, entry_tid=36486' 2026-02-19T20:33:32.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:32.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36486 2026-02-19T20:33:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 36486 2026-02-19T20:33:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:32.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:32.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:32.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:32.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:32.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:32.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:32.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:32.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:32.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:32.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:32.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:32.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:32.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:32.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:32.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:32.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:32.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:32.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=121, tag_tid=8, entry_tid=36821], [object_number=120, tag_tid=8, entry_tid=36820], [object_number=123, tag_tid=8, entry_tid=36819], [object_number=122, tag_tid=8, entry_tid=36818]]], state=connected] 2026-02-19T20:33:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:32.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:32.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=121, tag_tid=8, entry_tid=36821], [object_number=120, tag_tid=8, entry_tid=36820], [object_number=123, tag_tid=8, entry_tid=36819], [object_number=122, tag_tid=8, entry_tid=36818]]], state=connected] 2026-02-19T20:33:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:32.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=121, tag_tid=8, entry_tid=36821' 2026-02-19T20:33:32.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=121, tag_tid=8, entry_tid=36821' 2026-02-19T20:33:32.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=121, tag_tid=8, entry_tid=36821' '!=' 'object_number=122, tag_tid=8, entry_tid=36486' 2026-02-19T20:33:32.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:32.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=121, tag_tid=8, entry_tid=36821' = 'object_number=122, tag_tid=8, entry_tid=36486' 2026-02-19T20:33:32.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=121, tag_tid=8, entry_tid=36821' 2026-02-19T20:33:32.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:32.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:32.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:32.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=121, tag_tid=8, entry_tid=36821' 2026-02-19T20:33:32.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:32.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:32.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=121, tag_tid=8, entry_tid=36821' 2026-02-19T20:33:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36821 2026-02-19T20:33:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 36821 2026-02-19T20:33:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:33.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:33.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:33.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:33.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:33.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=37147], [object_number=122, tag_tid=8, entry_tid=37146], [object_number=121, tag_tid=8, entry_tid=37145], [object_number=120, tag_tid=8, entry_tid=37144]]], state=connected] 2026-02-19T20:33:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:33.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:33.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:33.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:33.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=37147], [object_number=122, tag_tid=8, entry_tid=37146], [object_number=121, tag_tid=8, entry_tid=37145], [object_number=120, tag_tid=8, entry_tid=37144]]], state=connected] 2026-02-19T20:33:33.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=123, tag_tid=8, entry_tid=37147' 2026-02-19T20:33:33.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=123, tag_tid=8, entry_tid=37147' 2026-02-19T20:33:33.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=37147' '!=' 'object_number=121, tag_tid=8, entry_tid=36821' 2026-02-19T20:33:33.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:33.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=37147' = 'object_number=121, tag_tid=8, entry_tid=36821' 2026-02-19T20:33:33.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=123, tag_tid=8, entry_tid=37147' 2026-02-19T20:33:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=37147' 2026-02-19T20:33:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=37147' 2026-02-19T20:33:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37147 2026-02-19T20:33:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 37147 2026-02-19T20:33:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:33.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:33.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:33.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:33.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:33.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:33.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:33.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:33.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:33.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:33.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:33.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:33.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:33.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:33.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:33.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:33.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:33.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:33.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:33.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:33.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=121, tag_tid=8, entry_tid=37445], [object_number=120, tag_tid=8, entry_tid=37444], [object_number=123, tag_tid=8, entry_tid=37443], [object_number=122, tag_tid=8, entry_tid=37442]]], state=connected] 2026-02-19T20:33:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:33.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:33.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:33.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:33.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=121, tag_tid=8, entry_tid=37445], [object_number=120, tag_tid=8, entry_tid=37444], [object_number=123, tag_tid=8, entry_tid=37443], [object_number=122, tag_tid=8, entry_tid=37442]]], state=connected] 2026-02-19T20:33:33.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=121, tag_tid=8, entry_tid=37445' 2026-02-19T20:33:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=121, tag_tid=8, entry_tid=37445' 2026-02-19T20:33:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=121, tag_tid=8, entry_tid=37445' '!=' 'object_number=123, tag_tid=8, entry_tid=37147' 2026-02-19T20:33:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=121, tag_tid=8, entry_tid=37445' = 'object_number=123, tag_tid=8, entry_tid=37147' 2026-02-19T20:33:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=121, tag_tid=8, entry_tid=37445' 2026-02-19T20:33:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:33.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:33.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:33.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:33.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=121, tag_tid=8, entry_tid=37445' 2026-02-19T20:33:33.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:33.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=121, tag_tid=8, entry_tid=37445' 2026-02-19T20:33:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:33.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37445 2026-02-19T20:33:33.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 37445 2026-02-19T20:33:33.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:33.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:33.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:33.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:34.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:34.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:34.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:34.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:34.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:34.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:34.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:34.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:34.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=37727], [object_number=122, tag_tid=8, entry_tid=37726], [object_number=121, tag_tid=8, entry_tid=37725], [object_number=120, tag_tid=8, entry_tid=37724]]], state=connected] 2026-02-19T20:33:34.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:34.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:34.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:34.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:34.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:34.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:34.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=37727], [object_number=122, tag_tid=8, entry_tid=37726], [object_number=121, tag_tid=8, entry_tid=37725], [object_number=120, tag_tid=8, entry_tid=37724]]], state=connected] 2026-02-19T20:33:34.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=123, tag_tid=8, entry_tid=37727' 2026-02-19T20:33:34.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=123, tag_tid=8, entry_tid=37727' 2026-02-19T20:33:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=37727' '!=' 'object_number=121, tag_tid=8, entry_tid=37445' 2026-02-19T20:33:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=37727' = 'object_number=121, tag_tid=8, entry_tid=37445' 2026-02-19T20:33:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=123, tag_tid=8, entry_tid=37727' 2026-02-19T20:33:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=37727' 2026-02-19T20:33:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:34.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=37727' 2026-02-19T20:33:34.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:34.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37727 2026-02-19T20:33:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 37727 2026-02-19T20:33:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:34.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:34.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:34.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:34.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:34.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:34.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:34.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:34.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:34.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:34.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:34.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:34.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:34.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:34.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:34.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:34.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:34.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:34.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:34.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:34.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:34.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:34.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:34.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:34.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:34.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:34.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:34.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:34.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:34.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:34.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:34.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:34.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:34.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:34.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:34.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:34.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:34.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:34.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:34.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=38044], [object_number=123, tag_tid=8, entry_tid=38043], [object_number=122, tag_tid=8, entry_tid=38042], [object_number=121, tag_tid=8, entry_tid=38041]]], state=connected] 2026-02-19T20:33:34.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:34.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:34.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:34.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:34.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=38044], [object_number=123, tag_tid=8, entry_tid=38043], [object_number=122, tag_tid=8, entry_tid=38042], [object_number=121, tag_tid=8, entry_tid=38041]]], state=connected] 2026-02-19T20:33:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=120, tag_tid=8, entry_tid=38044' 2026-02-19T20:33:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=120, tag_tid=8, entry_tid=38044' 2026-02-19T20:33:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=38044' '!=' 'object_number=123, tag_tid=8, entry_tid=37727' 2026-02-19T20:33:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=38044' = 'object_number=123, tag_tid=8, entry_tid=37727' 2026-02-19T20:33:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=120, tag_tid=8, entry_tid=38044' 2026-02-19T20:33:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=38044' 2026-02-19T20:33:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:34.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:34.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=38044' 2026-02-19T20:33:34.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:34.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38044 2026-02-19T20:33:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 38044 2026-02-19T20:33:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:34.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:34.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:34.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:34.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:34.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:34.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:34.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=121, tag_tid=8, entry_tid=38349], [object_number=120, tag_tid=8, entry_tid=38348], [object_number=123, tag_tid=8, entry_tid=38347], [object_number=122, tag_tid=8, entry_tid=38346]]], state=connected] 2026-02-19T20:33:35.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:35.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:35.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:35.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:35.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:35.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:35.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:35.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:35.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:35.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:35.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:35.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:35.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:35.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:35.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=121, tag_tid=8, entry_tid=38349], [object_number=120, tag_tid=8, entry_tid=38348], [object_number=123, tag_tid=8, entry_tid=38347], [object_number=122, tag_tid=8, entry_tid=38346]]], state=connected] 2026-02-19T20:33:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=121, tag_tid=8, entry_tid=38349' 2026-02-19T20:33:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=121, tag_tid=8, entry_tid=38349' 2026-02-19T20:33:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=121, tag_tid=8, entry_tid=38349' '!=' 'object_number=120, tag_tid=8, entry_tid=38044' 2026-02-19T20:33:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=121, tag_tid=8, entry_tid=38349' = 'object_number=120, tag_tid=8, entry_tid=38044' 2026-02-19T20:33:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=121, tag_tid=8, entry_tid=38349' 2026-02-19T20:33:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=121, tag_tid=8, entry_tid=38349' 2026-02-19T20:33:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=121, tag_tid=8, entry_tid=38349' 2026-02-19T20:33:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38349 2026-02-19T20:33:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 38349 2026-02-19T20:33:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:35.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:35.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:35.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:35.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:35.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:35.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:35.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:35.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:35.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:35.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:35.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:35.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:35.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:35.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=122, tag_tid=8, entry_tid=38726], [object_number=121, tag_tid=8, entry_tid=38725], [object_number=120, tag_tid=8, entry_tid=38724], [object_number=123, tag_tid=8, entry_tid=38723]]], state=connected] 2026-02-19T20:33:35.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:35.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:35.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:35.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:35.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:35.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:35.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:35.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:35.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:35.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:35.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:35.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:35.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 30 2026-02-19T20:33:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=122, tag_tid=8, entry_tid=38726], [object_number=121, tag_tid=8, entry_tid=38725], [object_number=120, tag_tid=8, entry_tid=38724], [object_number=123, tag_tid=8, entry_tid=38723]]], state=connected] 2026-02-19T20:33:35.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=122, tag_tid=8, entry_tid=38726' 2026-02-19T20:33:35.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=122, tag_tid=8, entry_tid=38726' 2026-02-19T20:33:35.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=122, tag_tid=8, entry_tid=38726' '!=' 'object_number=121, tag_tid=8, entry_tid=38349' 2026-02-19T20:33:35.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=122, tag_tid=8, entry_tid=38726' = 'object_number=121, tag_tid=8, entry_tid=38349' 2026-02-19T20:33:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=122, tag_tid=8, entry_tid=38726' 2026-02-19T20:33:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:35.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=122, tag_tid=8, entry_tid=38726' 2026-02-19T20:33:35.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:35.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=122, tag_tid=8, entry_tid=38726' 2026-02-19T20:33:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38726 2026-02-19T20:33:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 38726 2026-02-19T20:33:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:35.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:35.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:35.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:35.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:35.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:35.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:35.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:35.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:35.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:35.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:35.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:35.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:35.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:35.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=38910], [object_number=125, tag_tid=8, entry_tid=38909], [object_number=124, tag_tid=8, entry_tid=38908], [object_number=127, tag_tid=8, entry_tid=38907]]], state=connected] 2026-02-19T20:33:35.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:35.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:35.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:35.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:35.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:35.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:35.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:35.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:35.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:35.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:35.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:35.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:35.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:35.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=38910], [object_number=125, tag_tid=8, entry_tid=38909], [object_number=124, tag_tid=8, entry_tid=38908], [object_number=127, tag_tid=8, entry_tid=38907]]], state=connected] 2026-02-19T20:33:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=126, tag_tid=8, entry_tid=38910' 2026-02-19T20:33:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=126, tag_tid=8, entry_tid=38910' 2026-02-19T20:33:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=38910' '!=' 'object_number=122, tag_tid=8, entry_tid=38726' 2026-02-19T20:33:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=38910' = 'object_number=122, tag_tid=8, entry_tid=38726' 2026-02-19T20:33:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=126, tag_tid=8, entry_tid=38910' 2026-02-19T20:33:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=38910' 2026-02-19T20:33:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:35.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=38910' 2026-02-19T20:33:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38910 2026-02-19T20:33:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 38910 2026-02-19T20:33:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:36.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:36.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:36.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:36.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:36.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:36.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:36.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:36.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:36.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=39211], [object_number=126, tag_tid=8, entry_tid=39210], [object_number=125, tag_tid=8, entry_tid=39209], [object_number=124, tag_tid=8, entry_tid=39208]]], state=connected] 2026-02-19T20:33:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:36.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=39211], [object_number=126, tag_tid=8, entry_tid=39210], [object_number=125, tag_tid=8, entry_tid=39209], [object_number=124, tag_tid=8, entry_tid=39208]]], state=connected] 2026-02-19T20:33:36.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=127, tag_tid=8, entry_tid=39211' 2026-02-19T20:33:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=127, tag_tid=8, entry_tid=39211' 2026-02-19T20:33:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=39211' '!=' 'object_number=126, tag_tid=8, entry_tid=38910' 2026-02-19T20:33:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=39211' = 'object_number=126, tag_tid=8, entry_tid=38910' 2026-02-19T20:33:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=127, tag_tid=8, entry_tid=39211' 2026-02-19T20:33:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:36.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:36.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:36.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=39211' 2026-02-19T20:33:36.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:36.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:36.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:36.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:36.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:36.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:36.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=39211' 2026-02-19T20:33:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39211 2026-02-19T20:33:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 39211 2026-02-19T20:33:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:36.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:36.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:36.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:36.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:36.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:36.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:36.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:36.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:36.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:36.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=39554], [object_number=125, tag_tid=8, entry_tid=39553], [object_number=124, tag_tid=8, entry_tid=39552], [object_number=127, tag_tid=8, entry_tid=39551]]], state=connected] 2026-02-19T20:33:36.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:36.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:36.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:36.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:36.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:36.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=39554], [object_number=125, tag_tid=8, entry_tid=39553], [object_number=124, tag_tid=8, entry_tid=39552], [object_number=127, tag_tid=8, entry_tid=39551]]], state=connected] 2026-02-19T20:33:36.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=126, tag_tid=8, entry_tid=39554' 2026-02-19T20:33:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=126, tag_tid=8, entry_tid=39554' 2026-02-19T20:33:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=39554' '!=' 'object_number=127, tag_tid=8, entry_tid=39211' 2026-02-19T20:33:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=39554' = 'object_number=127, tag_tid=8, entry_tid=39211' 2026-02-19T20:33:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=126, tag_tid=8, entry_tid=39554' 2026-02-19T20:33:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:36.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=39554' 2026-02-19T20:33:36.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:36.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:36.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=39554' 2026-02-19T20:33:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:36.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39554 2026-02-19T20:33:36.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 39554 2026-02-19T20:33:36.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:36.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:36.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:37.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:37.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:37.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:37.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:37.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:37.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=124, tag_tid=8, entry_tid=39900], [object_number=127, tag_tid=8, entry_tid=39899], [object_number=126, tag_tid=8, entry_tid=39898], [object_number=125, tag_tid=8, entry_tid=39897]]], state=connected] 2026-02-19T20:33:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:37.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:37.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:37.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:37.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:37.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:37.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:37.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=124, tag_tid=8, entry_tid=39900], [object_number=127, tag_tid=8, entry_tid=39899], [object_number=126, tag_tid=8, entry_tid=39898], [object_number=125, tag_tid=8, entry_tid=39897]]], state=connected] 2026-02-19T20:33:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=124, tag_tid=8, entry_tid=39900' 2026-02-19T20:33:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=124, tag_tid=8, entry_tid=39900' 2026-02-19T20:33:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=124, tag_tid=8, entry_tid=39900' '!=' 'object_number=126, tag_tid=8, entry_tid=39554' 2026-02-19T20:33:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=124, tag_tid=8, entry_tid=39900' = 'object_number=126, tag_tid=8, entry_tid=39554' 2026-02-19T20:33:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=124, tag_tid=8, entry_tid=39900' 2026-02-19T20:33:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:37.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=124, tag_tid=8, entry_tid=39900' 2026-02-19T20:33:37.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:37.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=124, tag_tid=8, entry_tid=39900' 2026-02-19T20:33:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39900 2026-02-19T20:33:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 39900 2026-02-19T20:33:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:37.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:37.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:37.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:37.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:37.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:37.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:37.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:37.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:37.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:37.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:37.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:37.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:37.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:37.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:37.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:37.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=40226], [object_number=125, tag_tid=8, entry_tid=40225], [object_number=124, tag_tid=8, entry_tid=40224], [object_number=127, tag_tid=8, entry_tid=40223]]], state=connected] 2026-02-19T20:33:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:37.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:37.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:37.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:37.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:37.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:37.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:37.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:37.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:37.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=40226], [object_number=125, tag_tid=8, entry_tid=40225], [object_number=124, tag_tid=8, entry_tid=40224], [object_number=127, tag_tid=8, entry_tid=40223]]], state=connected] 2026-02-19T20:33:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=126, tag_tid=8, entry_tid=40226' 2026-02-19T20:33:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=126, tag_tid=8, entry_tid=40226' 2026-02-19T20:33:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=40226' '!=' 'object_number=124, tag_tid=8, entry_tid=39900' 2026-02-19T20:33:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=40226' = 'object_number=124, tag_tid=8, entry_tid=39900' 2026-02-19T20:33:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=126, tag_tid=8, entry_tid=40226' 2026-02-19T20:33:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:37.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=40226' 2026-02-19T20:33:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=40226' 2026-02-19T20:33:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40226 2026-02-19T20:33:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 40226 2026-02-19T20:33:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:37.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:37.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:37.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:37.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:37.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:37.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:37.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:37.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:37.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:37.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:37.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:37.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:37.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:37.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:37.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=124, tag_tid=8, entry_tid=40508], [object_number=127, tag_tid=8, entry_tid=40507], [object_number=126, tag_tid=8, entry_tid=40506], [object_number=125, tag_tid=8, entry_tid=40505]]], state=connected] 2026-02-19T20:33:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:38.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:38.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:38.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:38.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=124, tag_tid=8, entry_tid=40508], [object_number=127, tag_tid=8, entry_tid=40507], [object_number=126, tag_tid=8, entry_tid=40506], [object_number=125, tag_tid=8, entry_tid=40505]]], state=connected] 2026-02-19T20:33:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=124, tag_tid=8, entry_tid=40508' 2026-02-19T20:33:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=124, tag_tid=8, entry_tid=40508' 2026-02-19T20:33:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=124, tag_tid=8, entry_tid=40508' '!=' 'object_number=126, tag_tid=8, entry_tid=40226' 2026-02-19T20:33:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=124, tag_tid=8, entry_tid=40508' = 'object_number=126, tag_tid=8, entry_tid=40226' 2026-02-19T20:33:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=124, tag_tid=8, entry_tid=40508' 2026-02-19T20:33:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:38.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=124, tag_tid=8, entry_tid=40508' 2026-02-19T20:33:38.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:38.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:38.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:38.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:38.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=124, tag_tid=8, entry_tid=40508' 2026-02-19T20:33:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40508 2026-02-19T20:33:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 40508 2026-02-19T20:33:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:38.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:38.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=124, tag_tid=8, entry_tid=40804], [object_number=127, tag_tid=8, entry_tid=40803], [object_number=126, tag_tid=8, entry_tid=40802], [object_number=125, tag_tid=8, entry_tid=40801]]], state=connected] 2026-02-19T20:33:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:38.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:38.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:38.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:38.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=124, tag_tid=8, entry_tid=40804], [object_number=127, tag_tid=8, entry_tid=40803], [object_number=126, tag_tid=8, entry_tid=40802], [object_number=125, tag_tid=8, entry_tid=40801]]], state=connected] 2026-02-19T20:33:38.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=124, tag_tid=8, entry_tid=40804' 2026-02-19T20:33:38.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=124, tag_tid=8, entry_tid=40804' 2026-02-19T20:33:38.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=124, tag_tid=8, entry_tid=40804' '!=' 'object_number=124, tag_tid=8, entry_tid=40508' 2026-02-19T20:33:38.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:38.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=124, tag_tid=8, entry_tid=40804' = 'object_number=124, tag_tid=8, entry_tid=40508' 2026-02-19T20:33:38.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=124, tag_tid=8, entry_tid=40804' 2026-02-19T20:33:38.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:38.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:38.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:38.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=124, tag_tid=8, entry_tid=40804' 2026-02-19T20:33:38.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:38.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:38.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:38.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:38.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:38.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=124, tag_tid=8, entry_tid=40804' 2026-02-19T20:33:38.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:38.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40804 2026-02-19T20:33:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 40804 2026-02-19T20:33:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:38.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:38.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:38.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:38.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=41151], [object_number=126, tag_tid=8, entry_tid=41150], [object_number=125, tag_tid=8, entry_tid=41149], [object_number=124, tag_tid=8, entry_tid=41148]]], state=connected] 2026-02-19T20:33:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=41151], [object_number=126, tag_tid=8, entry_tid=41150], [object_number=125, tag_tid=8, entry_tid=41149], [object_number=124, tag_tid=8, entry_tid=41148]]], state=connected] 2026-02-19T20:33:38.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:38.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=127, tag_tid=8, entry_tid=41151' 2026-02-19T20:33:38.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=127, tag_tid=8, entry_tid=41151' 2026-02-19T20:33:38.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=41151' '!=' 'object_number=124, tag_tid=8, entry_tid=40804' 2026-02-19T20:33:38.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:38.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=41151' = 'object_number=124, tag_tid=8, entry_tid=40804' 2026-02-19T20:33:38.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=127, tag_tid=8, entry_tid=41151' 2026-02-19T20:33:38.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:38.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:38.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:38.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=41151' 2026-02-19T20:33:38.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:38.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:38.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:38.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:38.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=41151' 2026-02-19T20:33:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41151 2026-02-19T20:33:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 41151 2026-02-19T20:33:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:39.035 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.807899704808319) > weight (0.25) 2026-02-19T20:33:39.035 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3602530566506329) > weight (0.25) 2026-02-19T20:33:39.035 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8903356469999366) > weight (0.25) 2026-02-19T20:33:39.035 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4756120430014956) > weight (0.25) 2026-02-19T20:33:39.035 INFO:tasks.rbd_mirror_thrash:waiting for 78.1451095672555 secs before thrashing 2026-02-19T20:33:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:39.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:39.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:39.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:39.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:39.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:39.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:39.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:39.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=41509], [object_number=124, tag_tid=8, entry_tid=41508], [object_number=127, tag_tid=8, entry_tid=41507], [object_number=126, tag_tid=8, entry_tid=41506]]], state=connected] 2026-02-19T20:33:39.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:39.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:39.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:39.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:39.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:39.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=41509], [object_number=124, tag_tid=8, entry_tid=41508], [object_number=127, tag_tid=8, entry_tid=41507], [object_number=126, tag_tid=8, entry_tid=41506]]], state=connected] 2026-02-19T20:33:39.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=125, tag_tid=8, entry_tid=41509' 2026-02-19T20:33:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=125, tag_tid=8, entry_tid=41509' 2026-02-19T20:33:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=41509' '!=' 'object_number=127, tag_tid=8, entry_tid=41151' 2026-02-19T20:33:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=41509' = 'object_number=127, tag_tid=8, entry_tid=41151' 2026-02-19T20:33:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=125, tag_tid=8, entry_tid=41509' 2026-02-19T20:33:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:39.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:39.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=41509' 2026-02-19T20:33:39.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:39.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=41509' 2026-02-19T20:33:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41509 2026-02-19T20:33:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 41509 2026-02-19T20:33:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:39.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:39.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:39.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:39.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:39.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:39.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:39.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:39.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:39.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:39.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:39.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:39.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:39.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:39.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:39.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:39.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:39.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:39.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=41818], [object_number=125, tag_tid=8, entry_tid=41817], [object_number=124, tag_tid=8, entry_tid=41816], [object_number=127, tag_tid=8, entry_tid=41815]]], state=connected] 2026-02-19T20:33:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:39.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=41818], [object_number=125, tag_tid=8, entry_tid=41817], [object_number=124, tag_tid=8, entry_tid=41816], [object_number=127, tag_tid=8, entry_tid=41815]]], state=connected] 2026-02-19T20:33:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=126, tag_tid=8, entry_tid=41818' 2026-02-19T20:33:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=126, tag_tid=8, entry_tid=41818' 2026-02-19T20:33:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=41818' '!=' 'object_number=125, tag_tid=8, entry_tid=41509' 2026-02-19T20:33:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=41818' = 'object_number=125, tag_tid=8, entry_tid=41509' 2026-02-19T20:33:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=126, tag_tid=8, entry_tid=41818' 2026-02-19T20:33:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=41818' 2026-02-19T20:33:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=41818' 2026-02-19T20:33:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41818 2026-02-19T20:33:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 41818 2026-02-19T20:33:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:40.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:40.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:40.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=42129], [object_number=124, tag_tid=8, entry_tid=42128], [object_number=127, tag_tid=8, entry_tid=42127], [object_number=126, tag_tid=8, entry_tid=42126]]], state=connected] 2026-02-19T20:33:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=42129], [object_number=124, tag_tid=8, entry_tid=42128], [object_number=127, tag_tid=8, entry_tid=42127], [object_number=126, tag_tid=8, entry_tid=42126]]], state=connected] 2026-02-19T20:33:40.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=125, tag_tid=8, entry_tid=42129' 2026-02-19T20:33:40.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=125, tag_tid=8, entry_tid=42129' 2026-02-19T20:33:40.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=42129' '!=' 'object_number=126, tag_tid=8, entry_tid=41818' 2026-02-19T20:33:40.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:40.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=42129' = 'object_number=126, tag_tid=8, entry_tid=41818' 2026-02-19T20:33:40.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=125, tag_tid=8, entry_tid=42129' 2026-02-19T20:33:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=42129' 2026-02-19T20:33:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=42129' 2026-02-19T20:33:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42129 2026-02-19T20:33:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 42129 2026-02-19T20:33:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:40.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:40.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:40.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:40.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:40.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:40.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:40.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:40.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=42469], [object_number=124, tag_tid=8, entry_tid=42468], [object_number=127, tag_tid=8, entry_tid=42467], [object_number=126, tag_tid=8, entry_tid=42466]]], state=connected] 2026-02-19T20:33:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:40.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=42469], [object_number=124, tag_tid=8, entry_tid=42468], [object_number=127, tag_tid=8, entry_tid=42467], [object_number=126, tag_tid=8, entry_tid=42466]]], state=connected] 2026-02-19T20:33:40.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:40.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=125, tag_tid=8, entry_tid=42469' 2026-02-19T20:33:40.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=125, tag_tid=8, entry_tid=42469' 2026-02-19T20:33:40.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=42469' '!=' 'object_number=125, tag_tid=8, entry_tid=42129' 2026-02-19T20:33:40.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:40.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=42469' = 'object_number=125, tag_tid=8, entry_tid=42129' 2026-02-19T20:33:40.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=125, tag_tid=8, entry_tid=42469' 2026-02-19T20:33:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=42469' 2026-02-19T20:33:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=42469' 2026-02-19T20:33:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42469 2026-02-19T20:33:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 42469 2026-02-19T20:33:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:40.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:40.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:40.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:40.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:40.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:40.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:40.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:40.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:40.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:40.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:40.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:40.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:40.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:40.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:40.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:40.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:41.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=42737], [object_number=124, tag_tid=8, entry_tid=42736], [object_number=127, tag_tid=8, entry_tid=42735], [object_number=126, tag_tid=8, entry_tid=42734]]], state=connected] 2026-02-19T20:33:41.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:41.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:41.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:41.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:41.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=42737], [object_number=124, tag_tid=8, entry_tid=42736], [object_number=127, tag_tid=8, entry_tid=42735], [object_number=126, tag_tid=8, entry_tid=42734]]], state=connected] 2026-02-19T20:33:41.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=125, tag_tid=8, entry_tid=42737' 2026-02-19T20:33:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=125, tag_tid=8, entry_tid=42737' 2026-02-19T20:33:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=42737' '!=' 'object_number=125, tag_tid=8, entry_tid=42469' 2026-02-19T20:33:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=42737' = 'object_number=125, tag_tid=8, entry_tid=42469' 2026-02-19T20:33:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=125, tag_tid=8, entry_tid=42737' 2026-02-19T20:33:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=42737' 2026-02-19T20:33:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=42737' 2026-02-19T20:33:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42737 2026-02-19T20:33:41.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 42737 2026-02-19T20:33:41.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:41.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:41.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:41.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:41.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:41.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:41.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:41.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:41.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:41.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:41.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:41.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:41.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=43043], [object_number=126, tag_tid=8, entry_tid=43042], [object_number=125, tag_tid=8, entry_tid=43041], [object_number=124, tag_tid=8, entry_tid=43040]]], state=connected] 2026-02-19T20:33:41.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 31 2026-02-19T20:33:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=43043], [object_number=126, tag_tid=8, entry_tid=43042], [object_number=125, tag_tid=8, entry_tid=43041], [object_number=124, tag_tid=8, entry_tid=43040]]], state=connected] 2026-02-19T20:33:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=127, tag_tid=8, entry_tid=43043' 2026-02-19T20:33:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=127, tag_tid=8, entry_tid=43043' 2026-02-19T20:33:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=43043' '!=' 'object_number=125, tag_tid=8, entry_tid=42737' 2026-02-19T20:33:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=43043' = 'object_number=125, tag_tid=8, entry_tid=42737' 2026-02-19T20:33:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=127, tag_tid=8, entry_tid=43043' 2026-02-19T20:33:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:41.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=43043' 2026-02-19T20:33:41.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:41.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:41.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:41.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:41.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:41.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=43043' 2026-02-19T20:33:41.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:41.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43043 2026-02-19T20:33:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 43043 2026-02-19T20:33:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:41.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:41.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:41.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:41.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:41.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:41.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:41.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:41.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:41.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:41.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:41.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:41.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:41.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:41.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:41.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:41.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:41.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:41.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:41.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:41.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:41.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:41.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:41.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=43264], [object_number=131, tag_tid=8, entry_tid=43263], [object_number=130, tag_tid=8, entry_tid=43262], [object_number=129, tag_tid=8, entry_tid=43261]]], state=connected] 2026-02-19T20:33:41.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:41.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:41.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:41.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:41.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:41.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:41.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:41.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:41.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=43264], [object_number=131, tag_tid=8, entry_tid=43263], [object_number=130, tag_tid=8, entry_tid=43262], [object_number=129, tag_tid=8, entry_tid=43261]]], state=connected] 2026-02-19T20:33:41.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:41.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=128, tag_tid=8, entry_tid=43264' 2026-02-19T20:33:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=128, tag_tid=8, entry_tid=43264' 2026-02-19T20:33:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=43264' '!=' 'object_number=127, tag_tid=8, entry_tid=43043' 2026-02-19T20:33:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=43264' = 'object_number=127, tag_tid=8, entry_tid=43043' 2026-02-19T20:33:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=128, tag_tid=8, entry_tid=43264' 2026-02-19T20:33:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=43264' 2026-02-19T20:33:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:41.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=43264' 2026-02-19T20:33:41.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:41.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43264 2026-02-19T20:33:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 43264 2026-02-19T20:33:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=43595], [object_number=130, tag_tid=8, entry_tid=43594], [object_number=129, tag_tid=8, entry_tid=43593], [object_number=128, tag_tid=8, entry_tid=43592]]], state=connected] 2026-02-19T20:33:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=43595], [object_number=130, tag_tid=8, entry_tid=43594], [object_number=129, tag_tid=8, entry_tid=43593], [object_number=128, tag_tid=8, entry_tid=43592]]], state=connected] 2026-02-19T20:33:42.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=43595' 2026-02-19T20:33:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=43595' 2026-02-19T20:33:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=43595' '!=' 'object_number=128, tag_tid=8, entry_tid=43264' 2026-02-19T20:33:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=43595' = 'object_number=128, tag_tid=8, entry_tid=43264' 2026-02-19T20:33:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=43595' 2026-02-19T20:33:42.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:42.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:42.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:42.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:42.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=43595' 2026-02-19T20:33:42.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:42.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:42.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:42.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:42.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:42.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:42.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:42.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=43595' 2026-02-19T20:33:42.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:42.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43595 2026-02-19T20:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 43595 2026-02-19T20:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:42.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:42.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:42.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:42.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:42.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:42.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:42.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:42.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:42.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:42.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:42.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:42.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:42.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:42.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:42.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:42.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:42.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:42.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:42.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:42.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:42.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:42.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=43907], [object_number=130, tag_tid=8, entry_tid=43906], [object_number=129, tag_tid=8, entry_tid=43905], [object_number=128, tag_tid=8, entry_tid=43904]]], state=connected] 2026-02-19T20:33:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:42.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:42.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:42.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:42.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:42.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:42.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:42.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:42.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:42.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=43907], [object_number=130, tag_tid=8, entry_tid=43906], [object_number=129, tag_tid=8, entry_tid=43905], [object_number=128, tag_tid=8, entry_tid=43904]]], state=connected] 2026-02-19T20:33:42.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=43907' 2026-02-19T20:33:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=43907' 2026-02-19T20:33:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=43907' '!=' 'object_number=131, tag_tid=8, entry_tid=43595' 2026-02-19T20:33:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=43907' = 'object_number=131, tag_tid=8, entry_tid=43595' 2026-02-19T20:33:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=43907' 2026-02-19T20:33:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:42.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:42.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=43907' 2026-02-19T20:33:42.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:42.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:42.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:42.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=43907' 2026-02-19T20:33:42.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:42.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43907 2026-02-19T20:33:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 43907 2026-02-19T20:33:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:42.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:42.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:43.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=44206], [object_number=129, tag_tid=8, entry_tid=44205], [object_number=128, tag_tid=8, entry_tid=44204], [object_number=131, tag_tid=8, entry_tid=44203]]], state=connected] 2026-02-19T20:33:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=44206], [object_number=129, tag_tid=8, entry_tid=44205], [object_number=128, tag_tid=8, entry_tid=44204], [object_number=131, tag_tid=8, entry_tid=44203]]], state=connected] 2026-02-19T20:33:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=130, tag_tid=8, entry_tid=44206' 2026-02-19T20:33:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=130, tag_tid=8, entry_tid=44206' 2026-02-19T20:33:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=44206' '!=' 'object_number=131, tag_tid=8, entry_tid=43907' 2026-02-19T20:33:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=44206' = 'object_number=131, tag_tid=8, entry_tid=43907' 2026-02-19T20:33:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=130, tag_tid=8, entry_tid=44206' 2026-02-19T20:33:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:43.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=44206' 2026-02-19T20:33:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=44206' 2026-02-19T20:33:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44206 2026-02-19T20:33:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 44206 2026-02-19T20:33:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:43.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:43.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:43.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:43.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:43.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:43.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:43.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:43.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:43.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:43.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:43.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=44492], [object_number=131, tag_tid=8, entry_tid=44491], [object_number=130, tag_tid=8, entry_tid=44490], [object_number=129, tag_tid=8, entry_tid=44489]]], state=connected] 2026-02-19T20:33:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:43.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:43.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:43.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=44492], [object_number=131, tag_tid=8, entry_tid=44491], [object_number=130, tag_tid=8, entry_tid=44490], [object_number=129, tag_tid=8, entry_tid=44489]]], state=connected] 2026-02-19T20:33:43.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=128, tag_tid=8, entry_tid=44492' 2026-02-19T20:33:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=128, tag_tid=8, entry_tid=44492' 2026-02-19T20:33:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=44492' '!=' 'object_number=130, tag_tid=8, entry_tid=44206' 2026-02-19T20:33:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=44492' = 'object_number=130, tag_tid=8, entry_tid=44206' 2026-02-19T20:33:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=128, tag_tid=8, entry_tid=44492' 2026-02-19T20:33:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:43.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=44492' 2026-02-19T20:33:43.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:43.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:43.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:43.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:43.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:43.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:43.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:43.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=44492' 2026-02-19T20:33:43.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44492 2026-02-19T20:33:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 44492 2026-02-19T20:33:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:43.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:43.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:43.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:43.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:43.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:43.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:43.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:43.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:43.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:43.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:43.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:43.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:43.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:43.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:43.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=44793], [object_number=128, tag_tid=8, entry_tid=44792], [object_number=131, tag_tid=8, entry_tid=44791], [object_number=130, tag_tid=8, entry_tid=44790]]], state=connected] 2026-02-19T20:33:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=44793], [object_number=128, tag_tid=8, entry_tid=44792], [object_number=131, tag_tid=8, entry_tid=44791], [object_number=130, tag_tid=8, entry_tid=44790]]], state=connected] 2026-02-19T20:33:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=44793' 2026-02-19T20:33:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=44793' 2026-02-19T20:33:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=44793' '!=' 'object_number=128, tag_tid=8, entry_tid=44492' 2026-02-19T20:33:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=44793' = 'object_number=128, tag_tid=8, entry_tid=44492' 2026-02-19T20:33:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=129, tag_tid=8, entry_tid=44793' 2026-02-19T20:33:44.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=44793' 2026-02-19T20:33:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:44.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=44793' 2026-02-19T20:33:44.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:44.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44793 2026-02-19T20:33:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 44793 2026-02-19T20:33:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:44.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:44.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:44.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:44.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=45103], [object_number=130, tag_tid=8, entry_tid=45102], [object_number=129, tag_tid=8, entry_tid=45101], [object_number=128, tag_tid=8, entry_tid=45100]]], state=connected] 2026-02-19T20:33:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:44.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=45103], [object_number=130, tag_tid=8, entry_tid=45102], [object_number=129, tag_tid=8, entry_tid=45101], [object_number=128, tag_tid=8, entry_tid=45100]]], state=connected] 2026-02-19T20:33:44.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=45103' 2026-02-19T20:33:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=45103' 2026-02-19T20:33:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=45103' '!=' 'object_number=129, tag_tid=8, entry_tid=44793' 2026-02-19T20:33:44.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:44.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=45103' = 'object_number=129, tag_tid=8, entry_tid=44793' 2026-02-19T20:33:44.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=45103' 2026-02-19T20:33:44.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:44.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:44.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:44.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=45103' 2026-02-19T20:33:44.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:44.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:44.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:44.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:44.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:44.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=45103' 2026-02-19T20:33:44.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:44.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45103 2026-02-19T20:33:44.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 45103 2026-02-19T20:33:44.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:44.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:44.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:44.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:44.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:44.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:44.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:44.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:44.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:44.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=45448], [object_number=131, tag_tid=8, entry_tid=45447], [object_number=130, tag_tid=8, entry_tid=45446], [object_number=129, tag_tid=8, entry_tid=45445]]], state=connected] 2026-02-19T20:33:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=45448], [object_number=131, tag_tid=8, entry_tid=45447], [object_number=130, tag_tid=8, entry_tid=45446], [object_number=129, tag_tid=8, entry_tid=45445]]], state=connected] 2026-02-19T20:33:44.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=128, tag_tid=8, entry_tid=45448' 2026-02-19T20:33:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=128, tag_tid=8, entry_tid=45448' 2026-02-19T20:33:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=45448' '!=' 'object_number=131, tag_tid=8, entry_tid=45103' 2026-02-19T20:33:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=45448' = 'object_number=131, tag_tid=8, entry_tid=45103' 2026-02-19T20:33:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=128, tag_tid=8, entry_tid=45448' 2026-02-19T20:33:44.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:44.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:44.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:44.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=45448' 2026-02-19T20:33:44.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:44.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:44.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=45448' 2026-02-19T20:33:44.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:44.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45448 2026-02-19T20:33:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 45448 2026-02-19T20:33:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:45.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:45.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:45.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=45739], [object_number=130, tag_tid=8, entry_tid=45738], [object_number=129, tag_tid=8, entry_tid=45737], [object_number=128, tag_tid=8, entry_tid=45736]]], state=connected] 2026-02-19T20:33:45.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:45.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=45739], [object_number=130, tag_tid=8, entry_tid=45738], [object_number=129, tag_tid=8, entry_tid=45737], [object_number=128, tag_tid=8, entry_tid=45736]]], state=connected] 2026-02-19T20:33:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=45739' 2026-02-19T20:33:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=45739' 2026-02-19T20:33:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=45739' '!=' 'object_number=128, tag_tid=8, entry_tid=45448' 2026-02-19T20:33:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=45739' = 'object_number=128, tag_tid=8, entry_tid=45448' 2026-02-19T20:33:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=45739' 2026-02-19T20:33:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:45.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:45.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:45.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=45739' 2026-02-19T20:33:45.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:45.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:45.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:45.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:45.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:45.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=45739' 2026-02-19T20:33:45.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:45.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45739 2026-02-19T20:33:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 45739 2026-02-19T20:33:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:45.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:45.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:45.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:45.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:45.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:45.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:45.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:45.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:45.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:45.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:45.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:45.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=46019], [object_number=130, tag_tid=8, entry_tid=46018], [object_number=129, tag_tid=8, entry_tid=46017], [object_number=128, tag_tid=8, entry_tid=46016]]], state=connected] 2026-02-19T20:33:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=46019], [object_number=130, tag_tid=8, entry_tid=46018], [object_number=129, tag_tid=8, entry_tid=46017], [object_number=128, tag_tid=8, entry_tid=46016]]], state=connected] 2026-02-19T20:33:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=46019' 2026-02-19T20:33:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=46019' 2026-02-19T20:33:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=46019' '!=' 'object_number=131, tag_tid=8, entry_tid=45739' 2026-02-19T20:33:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=46019' = 'object_number=131, tag_tid=8, entry_tid=45739' 2026-02-19T20:33:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=46019' 2026-02-19T20:33:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:45.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:45.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:45.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:45.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=46019' 2026-02-19T20:33:45.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:45.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:45.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:45.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=46019' 2026-02-19T20:33:45.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:45.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46019 2026-02-19T20:33:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 46019 2026-02-19T20:33:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:46.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=46145], [object_number=128, tag_tid=8, entry_tid=46144], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:46.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:46.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:46.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:46.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:46.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=46145], [object_number=128, tag_tid=8, entry_tid=46144], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=46145' '!=' 'object_number=131, tag_tid=8, entry_tid=46019' 2026-02-19T20:33:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:33:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=46145' = 'object_number=131, tag_tid=8, entry_tid=46019' 2026-02-19T20:33:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=9 2026-02-19T20:33:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:33:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=8 2026-02-19T20:33:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:46.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:33:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:33:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:33:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46145 2026-02-19T20:33:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 9 = 8 -a 1 -le 46145 2026-02-19T20:33:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:33:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:33:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:46.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:46.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:46.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:46.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:46.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:46.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:46.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:46.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:46.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:46.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:46.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:46.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:46.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:46.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:46.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:46.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=46145], [object_number=128, tag_tid=8, entry_tid=46144], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:46.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:46.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=46145], [object_number=128, tag_tid=8, entry_tid=46144], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=46145' '!=' 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:46.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:33:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:47.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:47.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:47.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:47.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:47.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:47.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:47.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:47.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:47.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:47.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=46145], [object_number=128, tag_tid=8, entry_tid=46144], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:47.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:47.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:47.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:47.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:47.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:47.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=46145], [object_number=128, tag_tid=8, entry_tid=46144], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:47.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=46145' '!=' 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:33:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:48.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:48.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:48.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:48.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:48.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:48.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:48.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:48.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:48.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:48.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:48.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:48.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:48.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:48.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:48.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:48.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:48.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:48.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=46145], [object_number=128, tag_tid=8, entry_tid=46144], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:48.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:48.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:48.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:48.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:48.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=46145], [object_number=128, tag_tid=8, entry_tid=46144], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=46145' '!=' 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:33:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:49.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:49.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:49.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:49.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:49.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:49.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:50.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:50.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:50.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:50.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:50.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:50.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:50.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:50.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:50.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:50.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:50.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=46145], [object_number=128, tag_tid=8, entry_tid=46144], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:50.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=46145], [object_number=128, tag_tid=8, entry_tid=46144], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=46145' '!=' 'object_number=129, tag_tid=8, entry_tid=46145' 2026-02-19T20:33:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:33:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:33:52.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:33:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:33:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:33:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:33:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:33:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:33:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:33:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:33:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:33:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:33:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:33:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:33:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:33:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:33:52.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:33:52.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:33:52.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:33:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:33:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:33:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:33:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:33:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:33:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:33:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:33:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:33:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:33:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:33:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:33:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:33:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:33:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:33:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:33:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:33:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:33:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:33:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:33:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:33:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:33:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:33:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:33:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:33:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:52.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:33:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:33:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:33:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:33:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:33:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:33:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:33:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:33:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:33:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:33:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:52.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 32 2026-02-19T20:33:52.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 32 2026-02-19T20:33:52.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:33:52.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:52.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=129, tag_tid=9, entry_tid=1], [object_number=128, tag_tid=9, entry_tid=0], [object_number=131, tag_tid=8, entry_tid=46143], [object_number=130, tag_tid=8, entry_tid=46142]]], state=connected] 2026-02-19T20:33:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:33:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=129, tag_tid=9, entry_tid=1' -a 'object_number=129, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=9, entry_tid=1' 2026-02-19T20:33:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2026-02-19T20:33:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:33:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap3 2026-02-19T20:33:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:33:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:33:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:33:53.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2026-02-19T20:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 5 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:32:32 2026 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:33:23 2026 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:33:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:33:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:33:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:33:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:33:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n snap2-clean ']' 2026-02-19T20:33:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-19T20:33:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap2-clean 2026-02-19T20:33:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:33:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:33:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:33:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:33:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:33:53.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:33:53.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:33:54.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:33:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:33:54.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:33:54.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:54.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap3 2026-02-19T20:33:54.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:54.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:54.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap3 2026-02-19T20:33:54.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:33:54.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:33:54.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:33:54.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:33:54.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:33:55.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:33:55.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:33:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:33:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:33:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:33:56.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:33:56.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name=snap3-clean 2026-02-19T20:33:56.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap3-clean 2026-02-19T20:33:56.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:33:56.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap3-clean 2026-02-19T20:33:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2026-02-19T20:33:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:33:56.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror test 2026-02-19T20:33:56.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:33:56.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:33:56.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:33:56.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:33:56.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=11 2026-02-19T20:33:56.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:33:56.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-19T20:33:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 2948096 bytes 2026-02-19T20:33:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 61678592 bytes 2026-02-19T20:33:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 71513088 bytes 2026-02-19T20:33:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 92881920 bytes 2026-02-19T20:33:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 137273344 bytes 2026-02-19T20:33:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 179782656 bytes 2026-02-19T20:33:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 214113280 bytes 2026-02-19T20:33:59.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 240610304 bytes 2026-02-19T20:34:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 291495936 bytes 2026-02-19T20:34:01.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 390523904 bytes 2026-02-19T20:34:01.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 411532288 bytes 2026-02-19T20:34:02.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 446844928 bytes 2026-02-19T20:34:02.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 481960960 bytes 2026-02-19T20:34:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 502098944 bytes 2026-02-19T20:34:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 525900800 bytes 2026-02-19T20:34:04.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 579328000 bytes 2026-02-19T20:34:04.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 585908224 bytes 2026-02-19T20:34:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 626085888 bytes 2026-02-19T20:34:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 657510400 bytes 2026-02-19T20:34:06.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2000 IOs, 748734464 bytes 2026-02-19T20:34:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2100 IOs, 765274112 bytes 2026-02-19T20:34:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2200 IOs, 803367936 bytes 2026-02-19T20:34:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:34:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:34:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:34:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-19T20:34:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:34:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:34:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:34:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:34:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-19T20:34:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:34:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:34:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:34:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:34:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:34:08.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-19T20:34:08.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='test: 2026-02-19T20:34:08.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: global_id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:34:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+replaying 2026-02-19T20:34:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: replaying, {"bytes_per_second":11020095.2,"entries_behind_primary":1,"entries_per_second":688.87,"non_primary_position":{"entry_tid":46145,"object_number":129,"tag_tid":8},"primary_position":{"entry_tid":1,"object_number":129,"tag_tid":9},"seconds_until_synced":0} 2026-02-19T20:34:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: service: mirror.1 on trial151 2026-02-19T20:34:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:33:47 2026-02-19T20:34:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: peer_sites: 2026-02-19T20:34:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: name: cluster2 2026-02-19T20:34:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+stopped 2026-02-19T20:34:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: local image is primary 2026-02-19T20:34:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:33:47' 2026-02-19T20:34:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ state: .*up+replaying' 2026-02-19T20:34:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: state: up+replaying 2026-02-19T20:34:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ description: .*' 2026-02-19T20:34:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: description: replaying, {"bytes_per_second":11020095.2,"entries_behind_primary":1,"entries_per_second":688.87,"non_primary_position":{"entry_tid":46145,"object_number":129,"tag_tid":8},"primary_position":{"entry_tid":1,"object_number":129,"tag_tid":9},"seconds_until_synced":0} 2026-02-19T20:34:08.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:34:08.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:34:08.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^up+' 2026-02-19T20:34:08.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:34:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep 'service: *mirror..* on ' 2026-02-19T20:34:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: service: mirror.1 on trial151 2026-02-19T20:34:08.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update 2026-02-19T20:34:08.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-19T20:34:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:33:47' 2026-02-19T20:34:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-19 20:33:47' 2026-02-19T20:34:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:34:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'prev_last_update=2026-02-19 20:33:47' 2026-02-19T20:34:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:34:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:34:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-19T20:34:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='OKOKOK142064262mirror.1trial15120.3.0-5336-g802bf111falseOK42104271mirror.2trial15120.3.0-5336-g802bf111trueOK42144267mirror.3trial15120.3.0-5336-g802bf111falseOK43494352mirror.0trial15120.3.0-5336-g802bf111falseOKtest959deb13-de1a-48d9-a341-1f95416f8379up+replayingreplaying, {"bytes_per_second":11020095.2,"entries_behind_primary":1,"entries_per_second":688.87,"non_primary_position":{"entry_tid":46145,"object_number":129,"tag_tid":8},"primary_position":{"entry_tid":1,"object_number":129,"tag_tid":9},"seconds_until_synced":0}42064262mirror.1trial1512026-02-19 20:33:47cluster2a054b997-3cfb-48e6-978a-70d6a85e56deup+stoppedlocal image is primary2026-02-19 20:33:47' 2026-02-19T20:34:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update state 2026-02-19T20:34:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-19T20:34:08.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:33:47' 2026-02-19T20:34:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-19T20:34:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=up+replaying 2026-02-19T20:34:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:34:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep up+replaying 2026-02-19T20:34:08.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:34:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap4 2026-02-19T20:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap4 2026-02-19T20:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap4 2026-02-19T20:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:34:36.278 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9664874118650328) > weight (0.25) 2026-02-19T20:34:36.278 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9928310688809244) > weight (0.25) 2026-02-19T20:34:36.278 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5126721094888759) > weight (0.25) 2026-02-19T20:34:36.278 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-19T20:34:36.278 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-19T20:34:36.278 INFO:tasks.rbd_mirror_thrash:waiting for 5.108470948815605 secs before reviving daemons 2026-02-19T20:34:36.279 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T20:34:36.273+0000 7f2ac2fb1640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 19462) UID: 1000 2026-02-19T20:34:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:34:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:34:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:34:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:34:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:34:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:34:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:34:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:34:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:34:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:34:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:34:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:34:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:34:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:34:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:34:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:34:39.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:39.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:34:39.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:34:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:34:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:34:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:34:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:34:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:34:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:34:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:39.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:39.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:39.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:39.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:34:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:34:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:34:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:34:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:34:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:34:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:34:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:34:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:34:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:34:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:34:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:34:39.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:34:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:34:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:34:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:34:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:34:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:34:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:39.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:40.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:40.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:40.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:40.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:40.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 39 2026-02-19T20:34:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=158, tag_tid=11, entry_tid=28910], [object_number=157, tag_tid=11, entry_tid=28909], [object_number=156, tag_tid=11, entry_tid=28908], [object_number=159, tag_tid=11, entry_tid=28907]]], state=connected] 2026-02-19T20:34:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 39 2026-02-19T20:34:40.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=158, tag_tid=11, entry_tid=28910], [object_number=157, tag_tid=11, entry_tid=28909], [object_number=156, tag_tid=11, entry_tid=28908], [object_number=159, tag_tid=11, entry_tid=28907]]], state=connected] 2026-02-19T20:34:40.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=158, tag_tid=11, entry_tid=28910' 2026-02-19T20:34:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=158, tag_tid=11, entry_tid=28910' 2026-02-19T20:34:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=158, tag_tid=11, entry_tid=28910' '!=' '' 2026-02-19T20:34:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=158, tag_tid=11, entry_tid=28910' = '' 2026-02-19T20:34:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=158, tag_tid=11, entry_tid=28910' 2026-02-19T20:34:40.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:40.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:40.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:40.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:40.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=158, tag_tid=11, entry_tid=28910' 2026-02-19T20:34:40.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:40.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=158, tag_tid=11, entry_tid=28910' 2026-02-19T20:34:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28910 2026-02-19T20:34:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 28910 2026-02-19T20:34:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:40.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:40.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:40.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:40.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:40.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:40.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:40.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:40.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:40.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:40.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:40.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:40.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 39 2026-02-19T20:34:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=158, tag_tid=11, entry_tid=29202], [object_number=157, tag_tid=11, entry_tid=29201], [object_number=156, tag_tid=11, entry_tid=29200], [object_number=159, tag_tid=11, entry_tid=29199]]], state=connected] 2026-02-19T20:34:40.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:40.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 39 2026-02-19T20:34:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=158, tag_tid=11, entry_tid=29202], [object_number=157, tag_tid=11, entry_tid=29201], [object_number=156, tag_tid=11, entry_tid=29200], [object_number=159, tag_tid=11, entry_tid=29199]]], state=connected] 2026-02-19T20:34:40.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=158, tag_tid=11, entry_tid=29202' 2026-02-19T20:34:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=158, tag_tid=11, entry_tid=29202' 2026-02-19T20:34:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=158, tag_tid=11, entry_tid=29202' '!=' 'object_number=158, tag_tid=11, entry_tid=28910' 2026-02-19T20:34:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=158, tag_tid=11, entry_tid=29202' = 'object_number=158, tag_tid=11, entry_tid=28910' 2026-02-19T20:34:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=158, tag_tid=11, entry_tid=29202' 2026-02-19T20:34:40.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:40.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:40.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:40.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=158, tag_tid=11, entry_tid=29202' 2026-02-19T20:34:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:40.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=158, tag_tid=11, entry_tid=29202' 2026-02-19T20:34:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29202 2026-02-19T20:34:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 29202 2026-02-19T20:34:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:40.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:40.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:40.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:40.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:40.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 39 2026-02-19T20:34:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=29495], [object_number=158, tag_tid=11, entry_tid=29494], [object_number=157, tag_tid=11, entry_tid=29493], [object_number=156, tag_tid=11, entry_tid=29492]]], state=connected] 2026-02-19T20:34:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 39 2026-02-19T20:34:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=29495], [object_number=158, tag_tid=11, entry_tid=29494], [object_number=157, tag_tid=11, entry_tid=29493], [object_number=156, tag_tid=11, entry_tid=29492]]], state=connected] 2026-02-19T20:34:41.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=159, tag_tid=11, entry_tid=29495' 2026-02-19T20:34:41.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=159, tag_tid=11, entry_tid=29495' 2026-02-19T20:34:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=29495' '!=' 'object_number=158, tag_tid=11, entry_tid=29202' 2026-02-19T20:34:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=29495' = 'object_number=158, tag_tid=11, entry_tid=29202' 2026-02-19T20:34:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=159, tag_tid=11, entry_tid=29495' 2026-02-19T20:34:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=29495' 2026-02-19T20:34:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=29495' 2026-02-19T20:34:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29495 2026-02-19T20:34:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 29495 2026-02-19T20:34:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:41.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:41.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:41.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:41.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:41.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:41.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:41.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:41.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:41.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:41.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:41.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:41.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:41.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:41.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:41.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:41.387 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-19T20:34:41.387 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:34:41.388 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-19T20:34:41.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-19T20:34:41.388 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-19T20:34:41.390 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-19T20:34:41.390 INFO:tasks.rbd_mirror_thrash:waiting for 79.65176118833585 secs before thrashing 2026-02-19T20:34:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:41.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:41.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:41.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:41.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 39 2026-02-19T20:34:41.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:41.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:41.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:41.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=29863], [object_number=158, tag_tid=11, entry_tid=29862], [object_number=157, tag_tid=11, entry_tid=29861], [object_number=156, tag_tid=11, entry_tid=29860]]], state=connected] 2026-02-19T20:34:41.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:41.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 39 2026-02-19T20:34:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=29863], [object_number=158, tag_tid=11, entry_tid=29862], [object_number=157, tag_tid=11, entry_tid=29861], [object_number=156, tag_tid=11, entry_tid=29860]]], state=connected] 2026-02-19T20:34:41.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=159, tag_tid=11, entry_tid=29863' 2026-02-19T20:34:41.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=159, tag_tid=11, entry_tid=29863' 2026-02-19T20:34:41.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=29863' '!=' 'object_number=159, tag_tid=11, entry_tid=29495' 2026-02-19T20:34:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=29863' = 'object_number=159, tag_tid=11, entry_tid=29495' 2026-02-19T20:34:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=159, tag_tid=11, entry_tid=29863' 2026-02-19T20:34:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:41.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:41.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:41.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=29863' 2026-02-19T20:34:41.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:41.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:41.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=29863' 2026-02-19T20:34:41.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:41.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29863 2026-02-19T20:34:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 29863 2026-02-19T20:34:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:41.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:41.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:41.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:41.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:41.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:41.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:41.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 39 2026-02-19T20:34:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=156, tag_tid=11, entry_tid=30168], [object_number=159, tag_tid=11, entry_tid=30167], [object_number=158, tag_tid=11, entry_tid=30166], [object_number=157, tag_tid=11, entry_tid=30165]]], state=connected] 2026-02-19T20:34:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:41.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 39 2026-02-19T20:34:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=156, tag_tid=11, entry_tid=30168], [object_number=159, tag_tid=11, entry_tid=30167], [object_number=158, tag_tid=11, entry_tid=30166], [object_number=157, tag_tid=11, entry_tid=30165]]], state=connected] 2026-02-19T20:34:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=156, tag_tid=11, entry_tid=30168' 2026-02-19T20:34:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=156, tag_tid=11, entry_tid=30168' 2026-02-19T20:34:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=156, tag_tid=11, entry_tid=30168' '!=' 'object_number=159, tag_tid=11, entry_tid=29863' 2026-02-19T20:34:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=156, tag_tid=11, entry_tid=30168' = 'object_number=159, tag_tid=11, entry_tid=29863' 2026-02-19T20:34:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=156, tag_tid=11, entry_tid=30168' 2026-02-19T20:34:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=156, tag_tid=11, entry_tid=30168' 2026-02-19T20:34:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:41.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=156, tag_tid=11, entry_tid=30168' 2026-02-19T20:34:41.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:41.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:41.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30168 2026-02-19T20:34:41.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 30168 2026-02-19T20:34:41.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:41.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:41.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:42.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:42.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:42.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:42.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:42.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:42.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:42.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:42.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=163, tag_tid=11, entry_tid=30339], [object_number=162, tag_tid=11, entry_tid=30338], [object_number=161, tag_tid=11, entry_tid=30337], [object_number=160, tag_tid=11, entry_tid=30336]]], state=connected] 2026-02-19T20:34:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:42.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=163, tag_tid=11, entry_tid=30339], [object_number=162, tag_tid=11, entry_tid=30338], [object_number=161, tag_tid=11, entry_tid=30337], [object_number=160, tag_tid=11, entry_tid=30336]]], state=connected] 2026-02-19T20:34:42.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=163, tag_tid=11, entry_tid=30339' 2026-02-19T20:34:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=163, tag_tid=11, entry_tid=30339' 2026-02-19T20:34:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=163, tag_tid=11, entry_tid=30339' '!=' 'object_number=156, tag_tid=11, entry_tid=30168' 2026-02-19T20:34:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=163, tag_tid=11, entry_tid=30339' = 'object_number=156, tag_tid=11, entry_tid=30168' 2026-02-19T20:34:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=163, tag_tid=11, entry_tid=30339' 2026-02-19T20:34:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=163, tag_tid=11, entry_tid=30339' 2026-02-19T20:34:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:42.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=163, tag_tid=11, entry_tid=30339' 2026-02-19T20:34:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30339 2026-02-19T20:34:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 30339 2026-02-19T20:34:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:42.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=30634], [object_number=161, tag_tid=11, entry_tid=30633], [object_number=160, tag_tid=11, entry_tid=30632], [object_number=163, tag_tid=11, entry_tid=30631]]], state=connected] 2026-02-19T20:34:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:42.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:42.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=30634], [object_number=161, tag_tid=11, entry_tid=30633], [object_number=160, tag_tid=11, entry_tid=30632], [object_number=163, tag_tid=11, entry_tid=30631]]], state=connected] 2026-02-19T20:34:42.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:42.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=162, tag_tid=11, entry_tid=30634' 2026-02-19T20:34:42.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=162, tag_tid=11, entry_tid=30634' 2026-02-19T20:34:42.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=30634' '!=' 'object_number=163, tag_tid=11, entry_tid=30339' 2026-02-19T20:34:42.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:42.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=30634' = 'object_number=163, tag_tid=11, entry_tid=30339' 2026-02-19T20:34:42.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=162, tag_tid=11, entry_tid=30634' 2026-02-19T20:34:42.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:42.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:42.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:42.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=30634' 2026-02-19T20:34:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:42.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:42.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:42.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=30634' 2026-02-19T20:34:42.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:42.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:42.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30634 2026-02-19T20:34:42.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 30634 2026-02-19T20:34:42.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:42.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:42.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:43.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:43.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:43.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:43.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:43.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:43.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:43.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:43.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:43.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=161, tag_tid=11, entry_tid=30949], [object_number=160, tag_tid=11, entry_tid=30948], [object_number=163, tag_tid=11, entry_tid=30947], [object_number=162, tag_tid=11, entry_tid=30946]]], state=connected] 2026-02-19T20:34:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:43.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:43.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:43.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:43.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:43.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:43.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:43.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=161, tag_tid=11, entry_tid=30949], [object_number=160, tag_tid=11, entry_tid=30948], [object_number=163, tag_tid=11, entry_tid=30947], [object_number=162, tag_tid=11, entry_tid=30946]]], state=connected] 2026-02-19T20:34:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=161, tag_tid=11, entry_tid=30949' 2026-02-19T20:34:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=161, tag_tid=11, entry_tid=30949' 2026-02-19T20:34:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=161, tag_tid=11, entry_tid=30949' '!=' 'object_number=162, tag_tid=11, entry_tid=30634' 2026-02-19T20:34:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=161, tag_tid=11, entry_tid=30949' = 'object_number=162, tag_tid=11, entry_tid=30634' 2026-02-19T20:34:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=161, tag_tid=11, entry_tid=30949' 2026-02-19T20:34:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=161, tag_tid=11, entry_tid=30949' 2026-02-19T20:34:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=161, tag_tid=11, entry_tid=30949' 2026-02-19T20:34:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30949 2026-02-19T20:34:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 30949 2026-02-19T20:34:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:43.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:43.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:43.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:43.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:43.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:43.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:43.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:43.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:43.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:43.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:43.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:43.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:43.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:43.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:43.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=163, tag_tid=11, entry_tid=31243], [object_number=162, tag_tid=11, entry_tid=31242], [object_number=161, tag_tid=11, entry_tid=31241], [object_number=160, tag_tid=11, entry_tid=31240]]], state=connected] 2026-02-19T20:34:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:43.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:43.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:43.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:43.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:43.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:43.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:43.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:43.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:43.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:43.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:43.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:43.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=163, tag_tid=11, entry_tid=31243], [object_number=162, tag_tid=11, entry_tid=31242], [object_number=161, tag_tid=11, entry_tid=31241], [object_number=160, tag_tid=11, entry_tid=31240]]], state=connected] 2026-02-19T20:34:43.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:43.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=163, tag_tid=11, entry_tid=31243' 2026-02-19T20:34:43.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=163, tag_tid=11, entry_tid=31243' 2026-02-19T20:34:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=163, tag_tid=11, entry_tid=31243' '!=' 'object_number=161, tag_tid=11, entry_tid=30949' 2026-02-19T20:34:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=163, tag_tid=11, entry_tid=31243' = 'object_number=161, tag_tid=11, entry_tid=30949' 2026-02-19T20:34:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=163, tag_tid=11, entry_tid=31243' 2026-02-19T20:34:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=163, tag_tid=11, entry_tid=31243' 2026-02-19T20:34:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:43.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=163, tag_tid=11, entry_tid=31243' 2026-02-19T20:34:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31243 2026-02-19T20:34:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 31243 2026-02-19T20:34:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:43.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:43.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:43.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:43.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:43.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:43.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:43.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:43.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:43.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:43.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:43.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:44.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:44.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:44.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:44.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:44.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:44.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:44.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:44.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=163, tag_tid=11, entry_tid=31507], [object_number=162, tag_tid=11, entry_tid=31506], [object_number=161, tag_tid=11, entry_tid=31505], [object_number=160, tag_tid=11, entry_tid=31504]]], state=connected] 2026-02-19T20:34:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:44.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=163, tag_tid=11, entry_tid=31507], [object_number=162, tag_tid=11, entry_tid=31506], [object_number=161, tag_tid=11, entry_tid=31505], [object_number=160, tag_tid=11, entry_tid=31504]]], state=connected] 2026-02-19T20:34:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=163, tag_tid=11, entry_tid=31507' 2026-02-19T20:34:44.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=163, tag_tid=11, entry_tid=31507' 2026-02-19T20:34:44.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=163, tag_tid=11, entry_tid=31507' '!=' 'object_number=163, tag_tid=11, entry_tid=31243' 2026-02-19T20:34:44.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:44.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=163, tag_tid=11, entry_tid=31507' = 'object_number=163, tag_tid=11, entry_tid=31243' 2026-02-19T20:34:44.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=163, tag_tid=11, entry_tid=31507' 2026-02-19T20:34:44.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:44.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:44.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:44.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=163, tag_tid=11, entry_tid=31507' 2026-02-19T20:34:44.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:44.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:44.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=163, tag_tid=11, entry_tid=31507' 2026-02-19T20:34:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:44.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31507 2026-02-19T20:34:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 31507 2026-02-19T20:34:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:44.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:44.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:44.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:44.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:44.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:44.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:44.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:44.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:44.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:44.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:44.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=31782], [object_number=161, tag_tid=11, entry_tid=31781], [object_number=160, tag_tid=11, entry_tid=31780], [object_number=163, tag_tid=11, entry_tid=31779]]], state=connected] 2026-02-19T20:34:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:44.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:44.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:44.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:44.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:44.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:44.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=31782], [object_number=161, tag_tid=11, entry_tid=31781], [object_number=160, tag_tid=11, entry_tid=31780], [object_number=163, tag_tid=11, entry_tid=31779]]], state=connected] 2026-02-19T20:34:44.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=162, tag_tid=11, entry_tid=31782' 2026-02-19T20:34:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=162, tag_tid=11, entry_tid=31782' 2026-02-19T20:34:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=31782' '!=' 'object_number=163, tag_tid=11, entry_tid=31507' 2026-02-19T20:34:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=31782' = 'object_number=163, tag_tid=11, entry_tid=31507' 2026-02-19T20:34:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=162, tag_tid=11, entry_tid=31782' 2026-02-19T20:34:44.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:44.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:44.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:44.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=31782' 2026-02-19T20:34:44.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:44.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:44.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:44.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:44.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:44.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=31782' 2026-02-19T20:34:44.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:44.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31782 2026-02-19T20:34:44.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 31782 2026-02-19T20:34:44.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:44.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:44.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:44.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:44.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:44.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:44.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:44.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:44.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:44.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:44.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=32078], [object_number=161, tag_tid=11, entry_tid=32077], [object_number=160, tag_tid=11, entry_tid=32076], [object_number=163, tag_tid=11, entry_tid=32075]]], state=connected] 2026-02-19T20:34:44.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:44.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:44.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:44.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:44.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=32078], [object_number=161, tag_tid=11, entry_tid=32077], [object_number=160, tag_tid=11, entry_tid=32076], [object_number=163, tag_tid=11, entry_tid=32075]]], state=connected] 2026-02-19T20:34:44.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:44.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=162, tag_tid=11, entry_tid=32078' 2026-02-19T20:34:44.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=162, tag_tid=11, entry_tid=32078' 2026-02-19T20:34:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=32078' '!=' 'object_number=162, tag_tid=11, entry_tid=31782' 2026-02-19T20:34:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=32078' = 'object_number=162, tag_tid=11, entry_tid=31782' 2026-02-19T20:34:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=162, tag_tid=11, entry_tid=32078' 2026-02-19T20:34:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=32078' 2026-02-19T20:34:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:44.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:44.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:44.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=32078' 2026-02-19T20:34:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32078 2026-02-19T20:34:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 32078 2026-02-19T20:34:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:45.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:45.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:45.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:45.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:45.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:45.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:45.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:45.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:45.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:45.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:45.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:45.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=32422], [object_number=161, tag_tid=11, entry_tid=32421], [object_number=160, tag_tid=11, entry_tid=32420], [object_number=163, tag_tid=11, entry_tid=32419]]], state=connected] 2026-02-19T20:34:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=32422], [object_number=161, tag_tid=11, entry_tid=32421], [object_number=160, tag_tid=11, entry_tid=32420], [object_number=163, tag_tid=11, entry_tid=32419]]], state=connected] 2026-02-19T20:34:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=162, tag_tid=11, entry_tid=32422' 2026-02-19T20:34:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=162, tag_tid=11, entry_tid=32422' 2026-02-19T20:34:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=32422' '!=' 'object_number=162, tag_tid=11, entry_tid=32078' 2026-02-19T20:34:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=32422' = 'object_number=162, tag_tid=11, entry_tid=32078' 2026-02-19T20:34:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=162, tag_tid=11, entry_tid=32422' 2026-02-19T20:34:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=32422' 2026-02-19T20:34:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:45.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:45.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:45.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=32422' 2026-02-19T20:34:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32422 2026-02-19T20:34:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 32422 2026-02-19T20:34:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:45.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:45.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:45.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:45.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:45.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:45.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:45.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:45.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:45.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:45.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:45.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:45.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:45.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:45.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:45.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:45.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:45.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=32730], [object_number=161, tag_tid=11, entry_tid=32729], [object_number=160, tag_tid=11, entry_tid=32728], [object_number=163, tag_tid=11, entry_tid=32727]]], state=connected] 2026-02-19T20:34:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:45.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:45.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:45.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:45.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:45.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=32730], [object_number=161, tag_tid=11, entry_tid=32729], [object_number=160, tag_tid=11, entry_tid=32728], [object_number=163, tag_tid=11, entry_tid=32727]]], state=connected] 2026-02-19T20:34:45.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:45.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=162, tag_tid=11, entry_tid=32730' 2026-02-19T20:34:45.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=162, tag_tid=11, entry_tid=32730' 2026-02-19T20:34:45.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=32730' '!=' 'object_number=162, tag_tid=11, entry_tid=32422' 2026-02-19T20:34:45.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:45.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=32730' = 'object_number=162, tag_tid=11, entry_tid=32422' 2026-02-19T20:34:45.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=162, tag_tid=11, entry_tid=32730' 2026-02-19T20:34:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:45.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=32730' 2026-02-19T20:34:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=32730' 2026-02-19T20:34:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32730 2026-02-19T20:34:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 32730 2026-02-19T20:34:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:46.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:46.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:46.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:46.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=161, tag_tid=11, entry_tid=33037], [object_number=160, tag_tid=11, entry_tid=33036], [object_number=163, tag_tid=11, entry_tid=33035], [object_number=162, tag_tid=11, entry_tid=33034]]], state=connected] 2026-02-19T20:34:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:46.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=161, tag_tid=11, entry_tid=33037], [object_number=160, tag_tid=11, entry_tid=33036], [object_number=163, tag_tid=11, entry_tid=33035], [object_number=162, tag_tid=11, entry_tid=33034]]], state=connected] 2026-02-19T20:34:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=161, tag_tid=11, entry_tid=33037' 2026-02-19T20:34:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=161, tag_tid=11, entry_tid=33037' 2026-02-19T20:34:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=161, tag_tid=11, entry_tid=33037' '!=' 'object_number=162, tag_tid=11, entry_tid=32730' 2026-02-19T20:34:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=161, tag_tid=11, entry_tid=33037' = 'object_number=162, tag_tid=11, entry_tid=32730' 2026-02-19T20:34:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=161, tag_tid=11, entry_tid=33037' 2026-02-19T20:34:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:46.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=161, tag_tid=11, entry_tid=33037' 2026-02-19T20:34:46.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:46.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:46.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=161, tag_tid=11, entry_tid=33037' 2026-02-19T20:34:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33037 2026-02-19T20:34:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 33037 2026-02-19T20:34:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:46.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:46.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:46.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:46.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:46.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:46.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:46.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:46.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:46.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:46.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:46.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:46.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:46.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:46.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:46.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:46.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:46.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:46.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:46.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:46.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:46.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=160, tag_tid=11, entry_tid=33304], [object_number=163, tag_tid=11, entry_tid=33303], [object_number=162, tag_tid=11, entry_tid=33302], [object_number=161, tag_tid=11, entry_tid=33301]]], state=connected] 2026-02-19T20:34:46.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:46.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:46.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:46.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:46.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:46.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:46.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:46.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:46.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=160, tag_tid=11, entry_tid=33304], [object_number=163, tag_tid=11, entry_tid=33303], [object_number=162, tag_tid=11, entry_tid=33302], [object_number=161, tag_tid=11, entry_tid=33301]]], state=connected] 2026-02-19T20:34:46.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=160, tag_tid=11, entry_tid=33304' 2026-02-19T20:34:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=160, tag_tid=11, entry_tid=33304' 2026-02-19T20:34:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=160, tag_tid=11, entry_tid=33304' '!=' 'object_number=161, tag_tid=11, entry_tid=33037' 2026-02-19T20:34:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=160, tag_tid=11, entry_tid=33304' = 'object_number=161, tag_tid=11, entry_tid=33037' 2026-02-19T20:34:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=160, tag_tid=11, entry_tid=33304' 2026-02-19T20:34:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=160, tag_tid=11, entry_tid=33304' 2026-02-19T20:34:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:46.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=160, tag_tid=11, entry_tid=33304' 2026-02-19T20:34:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33304 2026-02-19T20:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 33304 2026-02-19T20:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:46.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:46.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:46.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:46.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:46.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:46.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:46.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=160, tag_tid=11, entry_tid=33568], [object_number=163, tag_tid=11, entry_tid=33567], [object_number=162, tag_tid=11, entry_tid=33566], [object_number=161, tag_tid=11, entry_tid=33565]]], state=connected] 2026-02-19T20:34:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:47.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:47.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:47.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:47.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=160, tag_tid=11, entry_tid=33568], [object_number=163, tag_tid=11, entry_tid=33567], [object_number=162, tag_tid=11, entry_tid=33566], [object_number=161, tag_tid=11, entry_tid=33565]]], state=connected] 2026-02-19T20:34:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=160, tag_tid=11, entry_tid=33568' 2026-02-19T20:34:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=160, tag_tid=11, entry_tid=33568' 2026-02-19T20:34:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=160, tag_tid=11, entry_tid=33568' '!=' 'object_number=160, tag_tid=11, entry_tid=33304' 2026-02-19T20:34:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=160, tag_tid=11, entry_tid=33568' = 'object_number=160, tag_tid=11, entry_tid=33304' 2026-02-19T20:34:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=160, tag_tid=11, entry_tid=33568' 2026-02-19T20:34:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:47.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:47.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=160, tag_tid=11, entry_tid=33568' 2026-02-19T20:34:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=160, tag_tid=11, entry_tid=33568' 2026-02-19T20:34:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33568 2026-02-19T20:34:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 33568 2026-02-19T20:34:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:47.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:47.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:47.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:47.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:47.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:47.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:47.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:47.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:47.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:47.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:47.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:47.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:47.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:47.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:47.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:47.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:47.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:47.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:47.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:47.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:47.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:47.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:47.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:47.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:47.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:47.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:47.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:47.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:47.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:47.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:47.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=33842], [object_number=161, tag_tid=11, entry_tid=33841], [object_number=160, tag_tid=11, entry_tid=33840], [object_number=163, tag_tid=11, entry_tid=33839]]], state=connected] 2026-02-19T20:34:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:47.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=162, tag_tid=11, entry_tid=33842], [object_number=161, tag_tid=11, entry_tid=33841], [object_number=160, tag_tid=11, entry_tid=33840], [object_number=163, tag_tid=11, entry_tid=33839]]], state=connected] 2026-02-19T20:34:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=162, tag_tid=11, entry_tid=33842' 2026-02-19T20:34:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=162, tag_tid=11, entry_tid=33842' 2026-02-19T20:34:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=33842' '!=' 'object_number=160, tag_tid=11, entry_tid=33568' 2026-02-19T20:34:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=162, tag_tid=11, entry_tid=33842' = 'object_number=160, tag_tid=11, entry_tid=33568' 2026-02-19T20:34:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=162, tag_tid=11, entry_tid=33842' 2026-02-19T20:34:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:47.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=33842' 2026-02-19T20:34:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:47.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:47.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:47.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:47.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=162, tag_tid=11, entry_tid=33842' 2026-02-19T20:34:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33842 2026-02-19T20:34:47.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 33842 2026-02-19T20:34:47.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:47.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:47.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:47.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:47.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:47.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:47.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:47.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:47.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:47.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:47.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=163, tag_tid=11, entry_tid=34107], [object_number=162, tag_tid=11, entry_tid=34106], [object_number=161, tag_tid=11, entry_tid=34105], [object_number=160, tag_tid=11, entry_tid=34104]]], state=connected] 2026-02-19T20:34:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:47.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=163, tag_tid=11, entry_tid=34107], [object_number=162, tag_tid=11, entry_tid=34106], [object_number=161, tag_tid=11, entry_tid=34105], [object_number=160, tag_tid=11, entry_tid=34104]]], state=connected] 2026-02-19T20:34:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=163, tag_tid=11, entry_tid=34107' 2026-02-19T20:34:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=163, tag_tid=11, entry_tid=34107' 2026-02-19T20:34:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=163, tag_tid=11, entry_tid=34107' '!=' 'object_number=162, tag_tid=11, entry_tid=33842' 2026-02-19T20:34:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=163, tag_tid=11, entry_tid=34107' = 'object_number=162, tag_tid=11, entry_tid=33842' 2026-02-19T20:34:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=163, tag_tid=11, entry_tid=34107' 2026-02-19T20:34:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=163, tag_tid=11, entry_tid=34107' 2026-02-19T20:34:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=163, tag_tid=11, entry_tid=34107' 2026-02-19T20:34:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34107 2026-02-19T20:34:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 34107 2026-02-19T20:34:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:48.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:48.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:48.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:48.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:48.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=161, tag_tid=11, entry_tid=34373], [object_number=160, tag_tid=11, entry_tid=34372], [object_number=163, tag_tid=11, entry_tid=34371], [object_number=162, tag_tid=11, entry_tid=34370]]], state=connected] 2026-02-19T20:34:48.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:48.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 40 2026-02-19T20:34:48.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:48.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:48.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:48.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=161, tag_tid=11, entry_tid=34373], [object_number=160, tag_tid=11, entry_tid=34372], [object_number=163, tag_tid=11, entry_tid=34371], [object_number=162, tag_tid=11, entry_tid=34370]]], state=connected] 2026-02-19T20:34:48.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=161, tag_tid=11, entry_tid=34373' 2026-02-19T20:34:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=161, tag_tid=11, entry_tid=34373' 2026-02-19T20:34:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=161, tag_tid=11, entry_tid=34373' '!=' 'object_number=163, tag_tid=11, entry_tid=34107' 2026-02-19T20:34:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=161, tag_tid=11, entry_tid=34373' = 'object_number=163, tag_tid=11, entry_tid=34107' 2026-02-19T20:34:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=161, tag_tid=11, entry_tid=34373' 2026-02-19T20:34:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:48.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=161, tag_tid=11, entry_tid=34373' 2026-02-19T20:34:48.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:48.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:48.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=161, tag_tid=11, entry_tid=34373' 2026-02-19T20:34:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34373 2026-02-19T20:34:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 34373 2026-02-19T20:34:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:48.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:48.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:48.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:48.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:48.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:48.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:48.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:48.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:48.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:48.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:48.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:48.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:48.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:48.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:48.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:48.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:48.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:48.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:48.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:48.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:48.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:48.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:48.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:48.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:48.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:48.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:48.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:48.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=34559], [object_number=166, tag_tid=11, entry_tid=34558], [object_number=165, tag_tid=11, entry_tid=34557], [object_number=164, tag_tid=11, entry_tid=34556]]], state=connected] 2026-02-19T20:34:48.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:48.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:48.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=34559], [object_number=166, tag_tid=11, entry_tid=34558], [object_number=165, tag_tid=11, entry_tid=34557], [object_number=164, tag_tid=11, entry_tid=34556]]], state=connected] 2026-02-19T20:34:48.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=167, tag_tid=11, entry_tid=34559' 2026-02-19T20:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=167, tag_tid=11, entry_tid=34559' 2026-02-19T20:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=34559' '!=' 'object_number=161, tag_tid=11, entry_tid=34373' 2026-02-19T20:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=34559' = 'object_number=161, tag_tid=11, entry_tid=34373' 2026-02-19T20:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=167, tag_tid=11, entry_tid=34559' 2026-02-19T20:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:48.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:48.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=34559' 2026-02-19T20:34:48.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:48.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:48.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:48.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:48.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:48.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=34559' 2026-02-19T20:34:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:48.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34559 2026-02-19T20:34:48.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 34559 2026-02-19T20:34:48.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:48.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:48.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=34843], [object_number=166, tag_tid=11, entry_tid=34842], [object_number=165, tag_tid=11, entry_tid=34841], [object_number=164, tag_tid=11, entry_tid=34840]]], state=connected] 2026-02-19T20:34:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:49.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=34843], [object_number=166, tag_tid=11, entry_tid=34842], [object_number=165, tag_tid=11, entry_tid=34841], [object_number=164, tag_tid=11, entry_tid=34840]]], state=connected] 2026-02-19T20:34:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=167, tag_tid=11, entry_tid=34843' 2026-02-19T20:34:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=167, tag_tid=11, entry_tid=34843' 2026-02-19T20:34:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=34843' '!=' 'object_number=167, tag_tid=11, entry_tid=34559' 2026-02-19T20:34:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=34843' = 'object_number=167, tag_tid=11, entry_tid=34559' 2026-02-19T20:34:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=167, tag_tid=11, entry_tid=34843' 2026-02-19T20:34:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=34843' 2026-02-19T20:34:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=34843' 2026-02-19T20:34:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34843 2026-02-19T20:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 34843 2026-02-19T20:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:49.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:49.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:49.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:49.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:49.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:49.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:49.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:49.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:49.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:49.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:49.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:49.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:49.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:49.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:49.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:49.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:49.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=35035], [object_number=166, tag_tid=11, entry_tid=35034], [object_number=165, tag_tid=11, entry_tid=35033], [object_number=164, tag_tid=11, entry_tid=35032]]], state=connected] 2026-02-19T20:34:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:49.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:49.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:49.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:49.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:49.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:49.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=35035], [object_number=166, tag_tid=11, entry_tid=35034], [object_number=165, tag_tid=11, entry_tid=35033], [object_number=164, tag_tid=11, entry_tid=35032]]], state=connected] 2026-02-19T20:34:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=167, tag_tid=11, entry_tid=35035' 2026-02-19T20:34:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=167, tag_tid=11, entry_tid=35035' 2026-02-19T20:34:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=35035' '!=' 'object_number=167, tag_tid=11, entry_tid=34843' 2026-02-19T20:34:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=35035' = 'object_number=167, tag_tid=11, entry_tid=34843' 2026-02-19T20:34:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=167, tag_tid=11, entry_tid=35035' 2026-02-19T20:34:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=35035' 2026-02-19T20:34:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:49.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:49.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:49.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:49.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:49.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=35035' 2026-02-19T20:34:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35035 2026-02-19T20:34:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 35035 2026-02-19T20:34:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:49.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:49.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:49.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:50.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:50.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:50.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:50.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:50.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:50.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:50.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:50.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:50.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=165, tag_tid=11, entry_tid=35349], [object_number=164, tag_tid=11, entry_tid=35348], [object_number=167, tag_tid=11, entry_tid=35347], [object_number=166, tag_tid=11, entry_tid=35346]]], state=connected] 2026-02-19T20:34:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:50.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:50.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:50.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:50.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:50.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=165, tag_tid=11, entry_tid=35349], [object_number=164, tag_tid=11, entry_tid=35348], [object_number=167, tag_tid=11, entry_tid=35347], [object_number=166, tag_tid=11, entry_tid=35346]]], state=connected] 2026-02-19T20:34:50.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=165, tag_tid=11, entry_tid=35349' 2026-02-19T20:34:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=165, tag_tid=11, entry_tid=35349' 2026-02-19T20:34:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=165, tag_tid=11, entry_tid=35349' '!=' 'object_number=167, tag_tid=11, entry_tid=35035' 2026-02-19T20:34:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=165, tag_tid=11, entry_tid=35349' = 'object_number=167, tag_tid=11, entry_tid=35035' 2026-02-19T20:34:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=165, tag_tid=11, entry_tid=35349' 2026-02-19T20:34:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=165, tag_tid=11, entry_tid=35349' 2026-02-19T20:34:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=165, tag_tid=11, entry_tid=35349' 2026-02-19T20:34:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35349 2026-02-19T20:34:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 35349 2026-02-19T20:34:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:50.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:50.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:50.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:50.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:50.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:50.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:50.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:50.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:50.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:50.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:50.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:50.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:50.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:50.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:50.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:50.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:50.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:50.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=166, tag_tid=11, entry_tid=35766], [object_number=165, tag_tid=11, entry_tid=35765], [object_number=164, tag_tid=11, entry_tid=35764], [object_number=167, tag_tid=11, entry_tid=35763]]], state=connected] 2026-02-19T20:34:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:50.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:50.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:50.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:50.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=166, tag_tid=11, entry_tid=35766], [object_number=165, tag_tid=11, entry_tid=35765], [object_number=164, tag_tid=11, entry_tid=35764], [object_number=167, tag_tid=11, entry_tid=35763]]], state=connected] 2026-02-19T20:34:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=166, tag_tid=11, entry_tid=35766' 2026-02-19T20:34:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=166, tag_tid=11, entry_tid=35766' 2026-02-19T20:34:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=166, tag_tid=11, entry_tid=35766' '!=' 'object_number=165, tag_tid=11, entry_tid=35349' 2026-02-19T20:34:50.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:50.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=166, tag_tid=11, entry_tid=35766' = 'object_number=165, tag_tid=11, entry_tid=35349' 2026-02-19T20:34:50.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=166, tag_tid=11, entry_tid=35766' 2026-02-19T20:34:50.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:50.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:50.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:50.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=166, tag_tid=11, entry_tid=35766' 2026-02-19T20:34:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:50.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:50.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:50.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=166, tag_tid=11, entry_tid=35766' 2026-02-19T20:34:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35766 2026-02-19T20:34:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 35766 2026-02-19T20:34:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:51.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:51.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:51.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:51.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:51.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:51.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:51.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:51.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=36099], [object_number=166, tag_tid=11, entry_tid=36098], [object_number=165, tag_tid=11, entry_tid=36097], [object_number=164, tag_tid=11, entry_tid=36096]]], state=connected] 2026-02-19T20:34:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:51.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=36099], [object_number=166, tag_tid=11, entry_tid=36098], [object_number=165, tag_tid=11, entry_tid=36097], [object_number=164, tag_tid=11, entry_tid=36096]]], state=connected] 2026-02-19T20:34:51.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=167, tag_tid=11, entry_tid=36099' 2026-02-19T20:34:51.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=167, tag_tid=11, entry_tid=36099' 2026-02-19T20:34:51.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=36099' '!=' 'object_number=166, tag_tid=11, entry_tid=35766' 2026-02-19T20:34:51.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:51.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=36099' = 'object_number=166, tag_tid=11, entry_tid=35766' 2026-02-19T20:34:51.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=167, tag_tid=11, entry_tid=36099' 2026-02-19T20:34:51.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:51.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:51.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:51.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:51.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=36099' 2026-02-19T20:34:51.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:51.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:51.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:51.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=36099' 2026-02-19T20:34:51.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:51.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36099 2026-02-19T20:34:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 36099 2026-02-19T20:34:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:51.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:51.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:51.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:51.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:51.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:51.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:51.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:51.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:51.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:51.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:51.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:51.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:51.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:51.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=164, tag_tid=11, entry_tid=36396], [object_number=167, tag_tid=11, entry_tid=36395], [object_number=166, tag_tid=11, entry_tid=36394], [object_number=165, tag_tid=11, entry_tid=36393]]], state=connected] 2026-02-19T20:34:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:51.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:51.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:51.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:51.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:51.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:51.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:51.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:51.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:51.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:51.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=164, tag_tid=11, entry_tid=36396], [object_number=167, tag_tid=11, entry_tid=36395], [object_number=166, tag_tid=11, entry_tid=36394], [object_number=165, tag_tid=11, entry_tid=36393]]], state=connected] 2026-02-19T20:34:51.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=164, tag_tid=11, entry_tid=36396' 2026-02-19T20:34:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=164, tag_tid=11, entry_tid=36396' 2026-02-19T20:34:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=164, tag_tid=11, entry_tid=36396' '!=' 'object_number=167, tag_tid=11, entry_tid=36099' 2026-02-19T20:34:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=164, tag_tid=11, entry_tid=36396' = 'object_number=167, tag_tid=11, entry_tid=36099' 2026-02-19T20:34:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=164, tag_tid=11, entry_tid=36396' 2026-02-19T20:34:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=164, tag_tid=11, entry_tid=36396' 2026-02-19T20:34:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=164, tag_tid=11, entry_tid=36396' 2026-02-19T20:34:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36396 2026-02-19T20:34:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 36396 2026-02-19T20:34:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:51.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:51.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:51.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:51.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:51.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:51.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:51.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:51.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:51.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:51.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:51.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:51.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:51.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:51.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:51.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:51.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:51.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:51.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:51.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:51.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:51.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:51.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:51.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=165, tag_tid=11, entry_tid=36685], [object_number=164, tag_tid=11, entry_tid=36684], [object_number=167, tag_tid=11, entry_tid=36683], [object_number=166, tag_tid=11, entry_tid=36682]]], state=connected] 2026-02-19T20:34:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:52.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:52.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:52.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:52.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:52.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:52.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:52.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:52.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:52.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:52.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:52.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:52.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:52.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=165, tag_tid=11, entry_tid=36685], [object_number=164, tag_tid=11, entry_tid=36684], [object_number=167, tag_tid=11, entry_tid=36683], [object_number=166, tag_tid=11, entry_tid=36682]]], state=connected] 2026-02-19T20:34:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=165, tag_tid=11, entry_tid=36685' 2026-02-19T20:34:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=165, tag_tid=11, entry_tid=36685' 2026-02-19T20:34:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=165, tag_tid=11, entry_tid=36685' '!=' 'object_number=164, tag_tid=11, entry_tid=36396' 2026-02-19T20:34:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=165, tag_tid=11, entry_tid=36685' = 'object_number=164, tag_tid=11, entry_tid=36396' 2026-02-19T20:34:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=165, tag_tid=11, entry_tid=36685' 2026-02-19T20:34:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:52.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=165, tag_tid=11, entry_tid=36685' 2026-02-19T20:34:52.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:52.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:52.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=165, tag_tid=11, entry_tid=36685' 2026-02-19T20:34:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36685 2026-02-19T20:34:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 36685 2026-02-19T20:34:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:52.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:52.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:52.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:52.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:52.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:52.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:52.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:52.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=36971], [object_number=166, tag_tid=11, entry_tid=36970], [object_number=165, tag_tid=11, entry_tid=36969], [object_number=164, tag_tid=11, entry_tid=36968]]], state=connected] 2026-02-19T20:34:52.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:52.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=36971], [object_number=166, tag_tid=11, entry_tid=36970], [object_number=165, tag_tid=11, entry_tid=36969], [object_number=164, tag_tid=11, entry_tid=36968]]], state=connected] 2026-02-19T20:34:52.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=167, tag_tid=11, entry_tid=36971' 2026-02-19T20:34:52.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=167, tag_tid=11, entry_tid=36971' 2026-02-19T20:34:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=36971' '!=' 'object_number=165, tag_tid=11, entry_tid=36685' 2026-02-19T20:34:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=36971' = 'object_number=165, tag_tid=11, entry_tid=36685' 2026-02-19T20:34:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=167, tag_tid=11, entry_tid=36971' 2026-02-19T20:34:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:52.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=36971' 2026-02-19T20:34:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:52.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:52.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=36971' 2026-02-19T20:34:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36971 2026-02-19T20:34:52.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 36971 2026-02-19T20:34:52.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:52.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:52.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:52.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:52.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:52.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:52.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:52.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:52.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:52.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:52.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:52.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:52.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:52.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:52.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=165, tag_tid=11, entry_tid=37273], [object_number=164, tag_tid=11, entry_tid=37272], [object_number=167, tag_tid=11, entry_tid=37271], [object_number=166, tag_tid=11, entry_tid=37270]]], state=connected] 2026-02-19T20:34:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:52.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:52.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:52.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:52.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:52.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:52.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=165, tag_tid=11, entry_tid=37273], [object_number=164, tag_tid=11, entry_tid=37272], [object_number=167, tag_tid=11, entry_tid=37271], [object_number=166, tag_tid=11, entry_tid=37270]]], state=connected] 2026-02-19T20:34:52.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:52.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=165, tag_tid=11, entry_tid=37273' 2026-02-19T20:34:52.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=165, tag_tid=11, entry_tid=37273' 2026-02-19T20:34:52.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=165, tag_tid=11, entry_tid=37273' '!=' 'object_number=167, tag_tid=11, entry_tid=36971' 2026-02-19T20:34:52.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:52.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=165, tag_tid=11, entry_tid=37273' = 'object_number=167, tag_tid=11, entry_tid=36971' 2026-02-19T20:34:52.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=165, tag_tid=11, entry_tid=37273' 2026-02-19T20:34:52.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:52.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:52.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:52.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=165, tag_tid=11, entry_tid=37273' 2026-02-19T20:34:52.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:52.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:52.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:52.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:52.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=165, tag_tid=11, entry_tid=37273' 2026-02-19T20:34:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37273 2026-02-19T20:34:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 37273 2026-02-19T20:34:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:52.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:52.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:53.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:53.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:53.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:53.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:53.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:53.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:53.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=37603], [object_number=166, tag_tid=11, entry_tid=37602], [object_number=165, tag_tid=11, entry_tid=37601], [object_number=164, tag_tid=11, entry_tid=37600]]], state=connected] 2026-02-19T20:34:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=37603], [object_number=166, tag_tid=11, entry_tid=37602], [object_number=165, tag_tid=11, entry_tid=37601], [object_number=164, tag_tid=11, entry_tid=37600]]], state=connected] 2026-02-19T20:34:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=167, tag_tid=11, entry_tid=37603' 2026-02-19T20:34:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=167, tag_tid=11, entry_tid=37603' 2026-02-19T20:34:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=37603' '!=' 'object_number=165, tag_tid=11, entry_tid=37273' 2026-02-19T20:34:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=37603' = 'object_number=165, tag_tid=11, entry_tid=37273' 2026-02-19T20:34:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=167, tag_tid=11, entry_tid=37603' 2026-02-19T20:34:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:53.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=37603' 2026-02-19T20:34:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:53.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:53.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:53.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:53.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=37603' 2026-02-19T20:34:53.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:53.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:53.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37603 2026-02-19T20:34:53.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 37603 2026-02-19T20:34:53.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:53.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:53.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:53.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:53.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:53.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:53.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:53.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:53.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=164, tag_tid=11, entry_tid=37884], [object_number=167, tag_tid=11, entry_tid=37883], [object_number=166, tag_tid=11, entry_tid=37882], [object_number=165, tag_tid=11, entry_tid=37881]]], state=connected] 2026-02-19T20:34:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:53.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:53.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:53.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:53.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:53.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=164, tag_tid=11, entry_tid=37884], [object_number=167, tag_tid=11, entry_tid=37883], [object_number=166, tag_tid=11, entry_tid=37882], [object_number=165, tag_tid=11, entry_tid=37881]]], state=connected] 2026-02-19T20:34:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=164, tag_tid=11, entry_tid=37884' 2026-02-19T20:34:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=164, tag_tid=11, entry_tid=37884' 2026-02-19T20:34:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=164, tag_tid=11, entry_tid=37884' '!=' 'object_number=167, tag_tid=11, entry_tid=37603' 2026-02-19T20:34:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=164, tag_tid=11, entry_tid=37884' = 'object_number=167, tag_tid=11, entry_tid=37603' 2026-02-19T20:34:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=164, tag_tid=11, entry_tid=37884' 2026-02-19T20:34:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=164, tag_tid=11, entry_tid=37884' 2026-02-19T20:34:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:53.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:53.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=164, tag_tid=11, entry_tid=37884' 2026-02-19T20:34:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:53.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37884 2026-02-19T20:34:53.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 37884 2026-02-19T20:34:53.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:53.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:53.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:53.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:53.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:53.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:53.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:53.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:53.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:53.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:53.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:53.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:54.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:54.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:54.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:54.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:54.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:54.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:54.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=164, tag_tid=11, entry_tid=38160], [object_number=167, tag_tid=11, entry_tid=38159], [object_number=166, tag_tid=11, entry_tid=38158], [object_number=165, tag_tid=11, entry_tid=38157]]], state=connected] 2026-02-19T20:34:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=164, tag_tid=11, entry_tid=38160], [object_number=167, tag_tid=11, entry_tid=38159], [object_number=166, tag_tid=11, entry_tid=38158], [object_number=165, tag_tid=11, entry_tid=38157]]], state=connected] 2026-02-19T20:34:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=164, tag_tid=11, entry_tid=38160' 2026-02-19T20:34:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=164, tag_tid=11, entry_tid=38160' 2026-02-19T20:34:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=164, tag_tid=11, entry_tid=38160' '!=' 'object_number=164, tag_tid=11, entry_tid=37884' 2026-02-19T20:34:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=164, tag_tid=11, entry_tid=38160' = 'object_number=164, tag_tid=11, entry_tid=37884' 2026-02-19T20:34:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=164, tag_tid=11, entry_tid=38160' 2026-02-19T20:34:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=164, tag_tid=11, entry_tid=38160' 2026-02-19T20:34:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:54.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=164, tag_tid=11, entry_tid=38160' 2026-02-19T20:34:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38160 2026-02-19T20:34:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 38160 2026-02-19T20:34:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:54.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:54.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:54.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:54.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:54.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:54.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:54.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:54.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:54.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:54.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=165, tag_tid=11, entry_tid=38437], [object_number=164, tag_tid=11, entry_tid=38436], [object_number=167, tag_tid=11, entry_tid=38435], [object_number=166, tag_tid=11, entry_tid=38434]]], state=connected] 2026-02-19T20:34:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:54.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:54.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:54.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:54.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:54.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:54.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:54.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:54.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=165, tag_tid=11, entry_tid=38437], [object_number=164, tag_tid=11, entry_tid=38436], [object_number=167, tag_tid=11, entry_tid=38435], [object_number=166, tag_tid=11, entry_tid=38434]]], state=connected] 2026-02-19T20:34:54.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=165, tag_tid=11, entry_tid=38437' 2026-02-19T20:34:54.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=165, tag_tid=11, entry_tid=38437' 2026-02-19T20:34:54.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=165, tag_tid=11, entry_tid=38437' '!=' 'object_number=164, tag_tid=11, entry_tid=38160' 2026-02-19T20:34:54.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:54.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=165, tag_tid=11, entry_tid=38437' = 'object_number=164, tag_tid=11, entry_tid=38160' 2026-02-19T20:34:54.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=165, tag_tid=11, entry_tid=38437' 2026-02-19T20:34:54.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:54.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:54.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:54.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=165, tag_tid=11, entry_tid=38437' 2026-02-19T20:34:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:54.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:54.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:54.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:54.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:54.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=165, tag_tid=11, entry_tid=38437' 2026-02-19T20:34:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:54.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38437 2026-02-19T20:34:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 38437 2026-02-19T20:34:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:54.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:54.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:54.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:54.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:54.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:54.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:54.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:54.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:54.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:54.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:54.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:54.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=38735], [object_number=166, tag_tid=11, entry_tid=38734], [object_number=165, tag_tid=11, entry_tid=38733], [object_number=164, tag_tid=11, entry_tid=38732]]], state=connected] 2026-02-19T20:34:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 41 2026-02-19T20:34:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=167, tag_tid=11, entry_tid=38735], [object_number=166, tag_tid=11, entry_tid=38734], [object_number=165, tag_tid=11, entry_tid=38733], [object_number=164, tag_tid=11, entry_tid=38732]]], state=connected] 2026-02-19T20:34:55.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=167, tag_tid=11, entry_tid=38735' 2026-02-19T20:34:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=167, tag_tid=11, entry_tid=38735' 2026-02-19T20:34:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=38735' '!=' 'object_number=165, tag_tid=11, entry_tid=38437' 2026-02-19T20:34:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=167, tag_tid=11, entry_tid=38735' = 'object_number=165, tag_tid=11, entry_tid=38437' 2026-02-19T20:34:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=167, tag_tid=11, entry_tid=38735' 2026-02-19T20:34:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:55.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=38735' 2026-02-19T20:34:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:55.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=167, tag_tid=11, entry_tid=38735' 2026-02-19T20:34:55.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:55.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38735 2026-02-19T20:34:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 38735 2026-02-19T20:34:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:55.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:55.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:55.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:55.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:55.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:55.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:55.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:55.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:55.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:55.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:55.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:55.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:55.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:55.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:55.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:55.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:55.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:55.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=168, tag_tid=11, entry_tid=38976], [object_number=171, tag_tid=11, entry_tid=38975], [object_number=170, tag_tid=11, entry_tid=38974], [object_number=169, tag_tid=11, entry_tid=38973]]], state=connected] 2026-02-19T20:34:55.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:55.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:55.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:55.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:55.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=168, tag_tid=11, entry_tid=38976], [object_number=171, tag_tid=11, entry_tid=38975], [object_number=170, tag_tid=11, entry_tid=38974], [object_number=169, tag_tid=11, entry_tid=38973]]], state=connected] 2026-02-19T20:34:55.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=168, tag_tid=11, entry_tid=38976' 2026-02-19T20:34:55.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=168, tag_tid=11, entry_tid=38976' 2026-02-19T20:34:55.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=168, tag_tid=11, entry_tid=38976' '!=' 'object_number=167, tag_tid=11, entry_tid=38735' 2026-02-19T20:34:55.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:55.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=168, tag_tid=11, entry_tid=38976' = 'object_number=167, tag_tid=11, entry_tid=38735' 2026-02-19T20:34:55.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=168, tag_tid=11, entry_tid=38976' 2026-02-19T20:34:55.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:55.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:55.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:55.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=168, tag_tid=11, entry_tid=38976' 2026-02-19T20:34:55.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:55.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:55.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:55.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:55.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:55.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=168, tag_tid=11, entry_tid=38976' 2026-02-19T20:34:55.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:55.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38976 2026-02-19T20:34:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 38976 2026-02-19T20:34:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:55.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:55.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:55.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:55.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:55.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:55.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:55.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:55.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:55.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:55.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:55.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:55.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:55.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:55.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:55.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:55.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:55.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:55.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:55.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:55.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:55.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:55.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:55.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:55.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:55.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:55.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:55.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:55.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:55.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:55.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:55.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:55.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:55.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:55.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:55.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:55.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:55.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:55.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:55.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:55.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:55.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:55.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:55.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:55.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:55.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:55.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:55.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:55.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:55.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:55.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=169, tag_tid=11, entry_tid=39277], [object_number=168, tag_tid=11, entry_tid=39276], [object_number=171, tag_tid=11, entry_tid=39275], [object_number=170, tag_tid=11, entry_tid=39274]]], state=connected] 2026-02-19T20:34:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:55.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:55.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:55.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:55.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:55.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:55.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=169, tag_tid=11, entry_tid=39277], [object_number=168, tag_tid=11, entry_tid=39276], [object_number=171, tag_tid=11, entry_tid=39275], [object_number=170, tag_tid=11, entry_tid=39274]]], state=connected] 2026-02-19T20:34:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:55.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=169, tag_tid=11, entry_tid=39277' 2026-02-19T20:34:55.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=169, tag_tid=11, entry_tid=39277' 2026-02-19T20:34:55.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=169, tag_tid=11, entry_tid=39277' '!=' 'object_number=168, tag_tid=11, entry_tid=38976' 2026-02-19T20:34:55.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:55.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=169, tag_tid=11, entry_tid=39277' = 'object_number=168, tag_tid=11, entry_tid=38976' 2026-02-19T20:34:55.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=169, tag_tid=11, entry_tid=39277' 2026-02-19T20:34:55.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:55.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:55.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:55.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=169, tag_tid=11, entry_tid=39277' 2026-02-19T20:34:55.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:55.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:55.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:55.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:55.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=169, tag_tid=11, entry_tid=39277' 2026-02-19T20:34:55.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:55.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39277 2026-02-19T20:34:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 39277 2026-02-19T20:34:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:56.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:56.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:56.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:56.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:56.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=168, tag_tid=11, entry_tid=39572], [object_number=171, tag_tid=11, entry_tid=39571], [object_number=170, tag_tid=11, entry_tid=39570], [object_number=169, tag_tid=11, entry_tid=39569]]], state=connected] 2026-02-19T20:34:56.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=168, tag_tid=11, entry_tid=39572], [object_number=171, tag_tid=11, entry_tid=39571], [object_number=170, tag_tid=11, entry_tid=39570], [object_number=169, tag_tid=11, entry_tid=39569]]], state=connected] 2026-02-19T20:34:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=168, tag_tid=11, entry_tid=39572' 2026-02-19T20:34:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=168, tag_tid=11, entry_tid=39572' 2026-02-19T20:34:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=168, tag_tid=11, entry_tid=39572' '!=' 'object_number=169, tag_tid=11, entry_tid=39277' 2026-02-19T20:34:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=168, tag_tid=11, entry_tid=39572' = 'object_number=169, tag_tid=11, entry_tid=39277' 2026-02-19T20:34:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=168, tag_tid=11, entry_tid=39572' 2026-02-19T20:34:56.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:56.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:56.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=168, tag_tid=11, entry_tid=39572' 2026-02-19T20:34:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:56.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:56.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:56.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:56.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=168, tag_tid=11, entry_tid=39572' 2026-02-19T20:34:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39572 2026-02-19T20:34:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 39572 2026-02-19T20:34:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:56.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:56.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:56.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:56.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:56.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:56.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:56.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:56.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:56.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:56.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:56.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:56.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:56.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:56.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:56.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:56.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:56.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:56.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:56.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:56.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:56.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:56.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:56.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:56.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:56.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:56.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:56.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=168, tag_tid=11, entry_tid=39864], [object_number=171, tag_tid=11, entry_tid=39863], [object_number=170, tag_tid=11, entry_tid=39862], [object_number=169, tag_tid=11, entry_tid=39861]]], state=connected] 2026-02-19T20:34:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:56.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:56.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:56.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:56.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=168, tag_tid=11, entry_tid=39864], [object_number=171, tag_tid=11, entry_tid=39863], [object_number=170, tag_tid=11, entry_tid=39862], [object_number=169, tag_tid=11, entry_tid=39861]]], state=connected] 2026-02-19T20:34:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:56.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=168, tag_tid=11, entry_tid=39864' 2026-02-19T20:34:56.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=168, tag_tid=11, entry_tid=39864' 2026-02-19T20:34:56.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=168, tag_tid=11, entry_tid=39864' '!=' 'object_number=168, tag_tid=11, entry_tid=39572' 2026-02-19T20:34:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=168, tag_tid=11, entry_tid=39864' = 'object_number=168, tag_tid=11, entry_tid=39572' 2026-02-19T20:34:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=168, tag_tid=11, entry_tid=39864' 2026-02-19T20:34:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:56.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=168, tag_tid=11, entry_tid=39864' 2026-02-19T20:34:56.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:56.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:56.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:56.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:56.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:56.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:56.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:56.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=168, tag_tid=11, entry_tid=39864' 2026-02-19T20:34:56.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:56.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39864 2026-02-19T20:34:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 39864 2026-02-19T20:34:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:57.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:57.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:57.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:57.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:57.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:57.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:57.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:57.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=170, tag_tid=11, entry_tid=40158], [object_number=169, tag_tid=11, entry_tid=40157], [object_number=168, tag_tid=11, entry_tid=40156], [object_number=171, tag_tid=11, entry_tid=40155]]], state=connected] 2026-02-19T20:34:57.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.182 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-19T20:34:57.182 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-19T20:34:57.182 INFO:tasks.rbd_mirror_thrash:waiting for 3.9534250302936025 secs before reviving daemons 2026-02-19T20:34:57.183 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T20:34:57.177+0000 7fba3bfbc640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 19480) UID: 1000 2026-02-19T20:34:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=170, tag_tid=11, entry_tid=40158], [object_number=169, tag_tid=11, entry_tid=40157], [object_number=168, tag_tid=11, entry_tid=40156], [object_number=171, tag_tid=11, entry_tid=40155]]], state=connected] 2026-02-19T20:34:57.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=170, tag_tid=11, entry_tid=40158' 2026-02-19T20:34:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=170, tag_tid=11, entry_tid=40158' 2026-02-19T20:34:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=170, tag_tid=11, entry_tid=40158' '!=' 'object_number=168, tag_tid=11, entry_tid=39864' 2026-02-19T20:34:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=170, tag_tid=11, entry_tid=40158' = 'object_number=168, tag_tid=11, entry_tid=39864' 2026-02-19T20:34:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=170, tag_tid=11, entry_tid=40158' 2026-02-19T20:34:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:57.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:57.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:57.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=170, tag_tid=11, entry_tid=40158' 2026-02-19T20:34:57.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:57.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:57.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=170, tag_tid=11, entry_tid=40158' 2026-02-19T20:34:57.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:57.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:57.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40158 2026-02-19T20:34:57.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 40158 2026-02-19T20:34:57.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:57.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:57.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:57.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:57.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:57.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:57.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:57.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:57.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:57.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:57.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:57.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:57.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:57.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:57.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:57.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:57.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:57.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:57.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:57.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:57.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:57.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:57.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:57.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:57.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:57.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:57.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:57.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:57.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:57.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:57.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:57.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:57.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=169, tag_tid=11, entry_tid=40457], [object_number=168, tag_tid=11, entry_tid=40456], [object_number=171, tag_tid=11, entry_tid=40455], [object_number=170, tag_tid=11, entry_tid=40454]]], state=connected] 2026-02-19T20:34:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:57.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:57.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:57.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:57.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=169, tag_tid=11, entry_tid=40457], [object_number=168, tag_tid=11, entry_tid=40456], [object_number=171, tag_tid=11, entry_tid=40455], [object_number=170, tag_tid=11, entry_tid=40454]]], state=connected] 2026-02-19T20:34:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=169, tag_tid=11, entry_tid=40457' 2026-02-19T20:34:57.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=169, tag_tid=11, entry_tid=40457' 2026-02-19T20:34:57.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=169, tag_tid=11, entry_tid=40457' '!=' 'object_number=170, tag_tid=11, entry_tid=40158' 2026-02-19T20:34:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=169, tag_tid=11, entry_tid=40457' = 'object_number=170, tag_tid=11, entry_tid=40158' 2026-02-19T20:34:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=169, tag_tid=11, entry_tid=40457' 2026-02-19T20:34:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:57.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:57.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=169, tag_tid=11, entry_tid=40457' 2026-02-19T20:34:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:57.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:57.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:57.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:57.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=169, tag_tid=11, entry_tid=40457' 2026-02-19T20:34:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40457 2026-02-19T20:34:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 40457 2026-02-19T20:34:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:57.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:57.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:57.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:57.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:57.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:57.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:57.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:57.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:57.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:57.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=170, tag_tid=11, entry_tid=40706], [object_number=169, tag_tid=11, entry_tid=40705], [object_number=168, tag_tid=11, entry_tid=40704], [object_number=171, tag_tid=11, entry_tid=40703]]], state=connected] 2026-02-19T20:34:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:58.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:58.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:58.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:58.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:58.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:58.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=170, tag_tid=11, entry_tid=40706], [object_number=169, tag_tid=11, entry_tid=40705], [object_number=168, tag_tid=11, entry_tid=40704], [object_number=171, tag_tid=11, entry_tid=40703]]], state=connected] 2026-02-19T20:34:58.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=170, tag_tid=11, entry_tid=40706' 2026-02-19T20:34:58.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=170, tag_tid=11, entry_tid=40706' 2026-02-19T20:34:58.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=170, tag_tid=11, entry_tid=40706' '!=' 'object_number=169, tag_tid=11, entry_tid=40457' 2026-02-19T20:34:58.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:58.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=170, tag_tid=11, entry_tid=40706' = 'object_number=169, tag_tid=11, entry_tid=40457' 2026-02-19T20:34:58.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=170, tag_tid=11, entry_tid=40706' 2026-02-19T20:34:58.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:58.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:58.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:58.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:58.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=170, tag_tid=11, entry_tid=40706' 2026-02-19T20:34:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:58.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:58.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:58.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:58.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:58.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:58.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=170, tag_tid=11, entry_tid=40706' 2026-02-19T20:34:58.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:58.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:58.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40706 2026-02-19T20:34:58.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 40706 2026-02-19T20:34:58.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:58.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:58.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:58.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=168, tag_tid=11, entry_tid=40964], [object_number=171, tag_tid=11, entry_tid=40963], [object_number=170, tag_tid=11, entry_tid=40962], [object_number=169, tag_tid=11, entry_tid=40961]]], state=connected] 2026-02-19T20:34:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:58.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:58.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:58.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=168, tag_tid=11, entry_tid=40964], [object_number=171, tag_tid=11, entry_tid=40963], [object_number=170, tag_tid=11, entry_tid=40962], [object_number=169, tag_tid=11, entry_tid=40961]]], state=connected] 2026-02-19T20:34:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=168, tag_tid=11, entry_tid=40964' 2026-02-19T20:34:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=168, tag_tid=11, entry_tid=40964' 2026-02-19T20:34:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=168, tag_tid=11, entry_tid=40964' '!=' 'object_number=170, tag_tid=11, entry_tid=40706' 2026-02-19T20:34:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=168, tag_tid=11, entry_tid=40964' = 'object_number=170, tag_tid=11, entry_tid=40706' 2026-02-19T20:34:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=168, tag_tid=11, entry_tid=40964' 2026-02-19T20:34:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:58.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:58.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=168, tag_tid=11, entry_tid=40964' 2026-02-19T20:34:58.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:58.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:58.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=168, tag_tid=11, entry_tid=40964' 2026-02-19T20:34:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40964 2026-02-19T20:34:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 40964 2026-02-19T20:34:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:58.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:58.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:58.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:58.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:58.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:58.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:58.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:58.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:59.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:59.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:59.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:59.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=170, tag_tid=11, entry_tid=41258], [object_number=169, tag_tid=11, entry_tid=41257], [object_number=168, tag_tid=11, entry_tid=41256], [object_number=171, tag_tid=11, entry_tid=41255]]], state=connected] 2026-02-19T20:34:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=170, tag_tid=11, entry_tid=41258], [object_number=169, tag_tid=11, entry_tid=41257], [object_number=168, tag_tid=11, entry_tid=41256], [object_number=171, tag_tid=11, entry_tid=41255]]], state=connected] 2026-02-19T20:34:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=170, tag_tid=11, entry_tid=41258' 2026-02-19T20:34:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=170, tag_tid=11, entry_tid=41258' 2026-02-19T20:34:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=170, tag_tid=11, entry_tid=41258' '!=' 'object_number=168, tag_tid=11, entry_tid=40964' 2026-02-19T20:34:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=170, tag_tid=11, entry_tid=41258' = 'object_number=168, tag_tid=11, entry_tid=40964' 2026-02-19T20:34:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=170, tag_tid=11, entry_tid=41258' 2026-02-19T20:34:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:59.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=170, tag_tid=11, entry_tid=41258' 2026-02-19T20:34:59.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:59.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:59.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:59.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:59.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=170, tag_tid=11, entry_tid=41258' 2026-02-19T20:34:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41258 2026-02-19T20:34:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 41258 2026-02-19T20:34:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:59.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:59.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:59.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:59.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:59.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:59.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:59.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:59.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=168, tag_tid=11, entry_tid=41688], [object_number=171, tag_tid=11, entry_tid=41687], [object_number=170, tag_tid=11, entry_tid=41686], [object_number=169, tag_tid=11, entry_tid=41685]]], state=connected] 2026-02-19T20:34:59.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=168, tag_tid=11, entry_tid=41688], [object_number=171, tag_tid=11, entry_tid=41687], [object_number=170, tag_tid=11, entry_tid=41686], [object_number=169, tag_tid=11, entry_tid=41685]]], state=connected] 2026-02-19T20:34:59.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=168, tag_tid=11, entry_tid=41688' 2026-02-19T20:34:59.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=168, tag_tid=11, entry_tid=41688' 2026-02-19T20:34:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=168, tag_tid=11, entry_tid=41688' '!=' 'object_number=170, tag_tid=11, entry_tid=41258' 2026-02-19T20:34:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=168, tag_tid=11, entry_tid=41688' = 'object_number=170, tag_tid=11, entry_tid=41258' 2026-02-19T20:34:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=168, tag_tid=11, entry_tid=41688' 2026-02-19T20:34:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:59.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:59.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=168, tag_tid=11, entry_tid=41688' 2026-02-19T20:34:59.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:59.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:59.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:59.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:59.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=168, tag_tid=11, entry_tid=41688' 2026-02-19T20:34:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41688 2026-02-19T20:34:59.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 41688 2026-02-19T20:34:59.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:34:59.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:34:59.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:34:59.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:34:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:34:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:34:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:34:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:34:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:34:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:34:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:34:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:59.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:34:59.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:34:59.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:34:59.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:34:59.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:34:59.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:34:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:34:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:34:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:34:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:34:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:34:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:34:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:34:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:34:59.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:59.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:34:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:34:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:34:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:34:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:34:59.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:34:59.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:34:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:34:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:34:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:34:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:34:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:34:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:34:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:34:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:34:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:34:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:34:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:34:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:34:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:34:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:34:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:34:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:34:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:34:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:34:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:34:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:34:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:34:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:34:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:34:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:34:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:34:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:59.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:59.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=171, tag_tid=11, entry_tid=42039], [object_number=170, tag_tid=11, entry_tid=42038], [object_number=169, tag_tid=11, entry_tid=42037], [object_number=168, tag_tid=11, entry_tid=42036]]], state=connected] 2026-02-19T20:34:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:34:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:34:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:34:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:34:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:34:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:34:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:34:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:34:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:34:59.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:34:59.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:34:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:34:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:34:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:34:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=171, tag_tid=11, entry_tid=42039], [object_number=170, tag_tid=11, entry_tid=42038], [object_number=169, tag_tid=11, entry_tid=42037], [object_number=168, tag_tid=11, entry_tid=42036]]], state=connected] 2026-02-19T20:34:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:34:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=171, tag_tid=11, entry_tid=42039' 2026-02-19T20:34:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=171, tag_tid=11, entry_tid=42039' 2026-02-19T20:34:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=171, tag_tid=11, entry_tid=42039' '!=' 'object_number=168, tag_tid=11, entry_tid=41688' 2026-02-19T20:34:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:34:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=171, tag_tid=11, entry_tid=42039' = 'object_number=168, tag_tid=11, entry_tid=41688' 2026-02-19T20:34:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=171, tag_tid=11, entry_tid=42039' 2026-02-19T20:34:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:34:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=171, tag_tid=11, entry_tid=42039' 2026-02-19T20:34:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:34:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:34:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:34:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:34:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=171, tag_tid=11, entry_tid=42039' 2026-02-19T20:34:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:34:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42039 2026-02-19T20:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 42039 2026-02-19T20:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:00.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:00.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:35:00.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=170, tag_tid=11, entry_tid=42398], [object_number=169, tag_tid=11, entry_tid=42397], [object_number=168, tag_tid=11, entry_tid=42396], [object_number=171, tag_tid=11, entry_tid=42395]]], state=connected] 2026-02-19T20:35:00.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:00.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:00.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:00.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:00.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:00.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:00.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:00.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:35:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=170, tag_tid=11, entry_tid=42398], [object_number=169, tag_tid=11, entry_tid=42397], [object_number=168, tag_tid=11, entry_tid=42396], [object_number=171, tag_tid=11, entry_tid=42395]]], state=connected] 2026-02-19T20:35:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=170, tag_tid=11, entry_tid=42398' 2026-02-19T20:35:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=170, tag_tid=11, entry_tid=42398' 2026-02-19T20:35:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=170, tag_tid=11, entry_tid=42398' '!=' 'object_number=171, tag_tid=11, entry_tid=42039' 2026-02-19T20:35:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=170, tag_tid=11, entry_tid=42398' = 'object_number=171, tag_tid=11, entry_tid=42039' 2026-02-19T20:35:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=170, tag_tid=11, entry_tid=42398' 2026-02-19T20:35:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=170, tag_tid=11, entry_tid=42398' 2026-02-19T20:35:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=170, tag_tid=11, entry_tid=42398' 2026-02-19T20:35:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42398 2026-02-19T20:35:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 42398 2026-02-19T20:35:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:00.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:35:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=170, tag_tid=11, entry_tid=42898], [object_number=169, tag_tid=11, entry_tid=42897], [object_number=168, tag_tid=11, entry_tid=42896], [object_number=171, tag_tid=11, entry_tid=42895]]], state=connected] 2026-02-19T20:35:01.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 42 2026-02-19T20:35:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=170, tag_tid=11, entry_tid=42898], [object_number=169, tag_tid=11, entry_tid=42897], [object_number=168, tag_tid=11, entry_tid=42896], [object_number=171, tag_tid=11, entry_tid=42895]]], state=connected] 2026-02-19T20:35:01.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=170, tag_tid=11, entry_tid=42898' 2026-02-19T20:35:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=170, tag_tid=11, entry_tid=42898' 2026-02-19T20:35:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=170, tag_tid=11, entry_tid=42898' '!=' 'object_number=170, tag_tid=11, entry_tid=42398' 2026-02-19T20:35:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=170, tag_tid=11, entry_tid=42898' = 'object_number=170, tag_tid=11, entry_tid=42398' 2026-02-19T20:35:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=170, tag_tid=11, entry_tid=42898' 2026-02-19T20:35:01.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:01.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:01.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:01.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=170, tag_tid=11, entry_tid=42898' 2026-02-19T20:35:01.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:01.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=170, tag_tid=11, entry_tid=42898' 2026-02-19T20:35:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42898 2026-02-19T20:35:01.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 42898 2026-02-19T20:35:01.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:01.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:01.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:01.136 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-19T20:35:01.137 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:35:01.137 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-19T20:35:01.137 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-19T20:35:01.138 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-19T20:35:01.140 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-19T20:35:01.140 INFO:tasks.rbd_mirror_thrash:waiting for 73.96025676868749 secs before thrashing 2026-02-19T20:35:01.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:01.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:01.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:01.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:01.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:01.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:01.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:01.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:01.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:01.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:01.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:01.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:01.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:01.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:01.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=175, tag_tid=11, entry_tid=43099], [object_number=174, tag_tid=11, entry_tid=43098], [object_number=173, tag_tid=11, entry_tid=43097], [object_number=172, tag_tid=11, entry_tid=43096]]], state=connected] 2026-02-19T20:35:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:01.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=175, tag_tid=11, entry_tid=43099], [object_number=174, tag_tid=11, entry_tid=43098], [object_number=173, tag_tid=11, entry_tid=43097], [object_number=172, tag_tid=11, entry_tid=43096]]], state=connected] 2026-02-19T20:35:01.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=175, tag_tid=11, entry_tid=43099' 2026-02-19T20:35:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=175, tag_tid=11, entry_tid=43099' 2026-02-19T20:35:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=175, tag_tid=11, entry_tid=43099' '!=' 'object_number=170, tag_tid=11, entry_tid=42898' 2026-02-19T20:35:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=175, tag_tid=11, entry_tid=43099' = 'object_number=170, tag_tid=11, entry_tid=42898' 2026-02-19T20:35:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=175, tag_tid=11, entry_tid=43099' 2026-02-19T20:35:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:01.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:01.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=175, tag_tid=11, entry_tid=43099' 2026-02-19T20:35:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=175, tag_tid=11, entry_tid=43099' 2026-02-19T20:35:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43099 2026-02-19T20:35:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 43099 2026-02-19T20:35:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:01.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:01.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:01.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:01.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:01.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:01.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:01.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:01.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:01.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:01.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:01.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:01.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:01.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:01.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:01.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:01.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=174, tag_tid=11, entry_tid=43362], [object_number=173, tag_tid=11, entry_tid=43361], [object_number=172, tag_tid=11, entry_tid=43360], [object_number=175, tag_tid=11, entry_tid=43359]]], state=connected] 2026-02-19T20:35:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:01.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:01.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:01.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:01.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:01.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:01.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:01.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:01.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:01.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:01.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:01.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=174, tag_tid=11, entry_tid=43362], [object_number=173, tag_tid=11, entry_tid=43361], [object_number=172, tag_tid=11, entry_tid=43360], [object_number=175, tag_tid=11, entry_tid=43359]]], state=connected] 2026-02-19T20:35:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=174, tag_tid=11, entry_tid=43362' 2026-02-19T20:35:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=174, tag_tid=11, entry_tid=43362' 2026-02-19T20:35:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=174, tag_tid=11, entry_tid=43362' '!=' 'object_number=175, tag_tid=11, entry_tid=43099' 2026-02-19T20:35:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=174, tag_tid=11, entry_tid=43362' = 'object_number=175, tag_tid=11, entry_tid=43099' 2026-02-19T20:35:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=174, tag_tid=11, entry_tid=43362' 2026-02-19T20:35:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:01.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:01.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:01.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=174, tag_tid=11, entry_tid=43362' 2026-02-19T20:35:01.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:01.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:01.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:01.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:01.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=174, tag_tid=11, entry_tid=43362' 2026-02-19T20:35:01.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:01.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43362 2026-02-19T20:35:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 43362 2026-02-19T20:35:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:02.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:02.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:02.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:02.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:02.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:02.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:02.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:02.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:02.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:02.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:02.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:02.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:02.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:02.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=175, tag_tid=11, entry_tid=43667], [object_number=174, tag_tid=11, entry_tid=43666], [object_number=173, tag_tid=11, entry_tid=43665], [object_number=172, tag_tid=11, entry_tid=43664]]], state=connected] 2026-02-19T20:35:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=175, tag_tid=11, entry_tid=43667], [object_number=174, tag_tid=11, entry_tid=43666], [object_number=173, tag_tid=11, entry_tid=43665], [object_number=172, tag_tid=11, entry_tid=43664]]], state=connected] 2026-02-19T20:35:02.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=175, tag_tid=11, entry_tid=43667' 2026-02-19T20:35:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=175, tag_tid=11, entry_tid=43667' 2026-02-19T20:35:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=175, tag_tid=11, entry_tid=43667' '!=' 'object_number=174, tag_tid=11, entry_tid=43362' 2026-02-19T20:35:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:02.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=175, tag_tid=11, entry_tid=43667' = 'object_number=174, tag_tid=11, entry_tid=43362' 2026-02-19T20:35:02.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=175, tag_tid=11, entry_tid=43667' 2026-02-19T20:35:02.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=175, tag_tid=11, entry_tid=43667' 2026-02-19T20:35:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=175, tag_tid=11, entry_tid=43667' 2026-02-19T20:35:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43667 2026-02-19T20:35:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 43667 2026-02-19T20:35:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:02.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:02.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:02.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:02.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:02.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:02.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:02.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:02.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:02.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:02.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:02.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:02.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:02.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:02.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:02.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:02.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:02.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:02.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:02.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:02.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=172, tag_tid=11, entry_tid=43944], [object_number=175, tag_tid=11, entry_tid=43943], [object_number=174, tag_tid=11, entry_tid=43942], [object_number=173, tag_tid=11, entry_tid=43941]]], state=connected] 2026-02-19T20:35:02.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:02.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:02.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:02.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:02.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:02.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:02.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:02.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=172, tag_tid=11, entry_tid=43944], [object_number=175, tag_tid=11, entry_tid=43943], [object_number=174, tag_tid=11, entry_tid=43942], [object_number=173, tag_tid=11, entry_tid=43941]]], state=connected] 2026-02-19T20:35:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=172, tag_tid=11, entry_tid=43944' 2026-02-19T20:35:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=172, tag_tid=11, entry_tid=43944' 2026-02-19T20:35:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=172, tag_tid=11, entry_tid=43944' '!=' 'object_number=175, tag_tid=11, entry_tid=43667' 2026-02-19T20:35:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=172, tag_tid=11, entry_tid=43944' = 'object_number=175, tag_tid=11, entry_tid=43667' 2026-02-19T20:35:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=172, tag_tid=11, entry_tid=43944' 2026-02-19T20:35:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:02.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=172, tag_tid=11, entry_tid=43944' 2026-02-19T20:35:02.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:02.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:02.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:02.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:02.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:02.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=172, tag_tid=11, entry_tid=43944' 2026-02-19T20:35:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43944 2026-02-19T20:35:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 43944 2026-02-19T20:35:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:03.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:03.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:03.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:03.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=173, tag_tid=11, entry_tid=44261], [object_number=172, tag_tid=11, entry_tid=44260], [object_number=175, tag_tid=11, entry_tid=44259], [object_number=174, tag_tid=11, entry_tid=44258]]], state=connected] 2026-02-19T20:35:03.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:03.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:03.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:03.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:03.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:03.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:03.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=173, tag_tid=11, entry_tid=44261], [object_number=172, tag_tid=11, entry_tid=44260], [object_number=175, tag_tid=11, entry_tid=44259], [object_number=174, tag_tid=11, entry_tid=44258]]], state=connected] 2026-02-19T20:35:03.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=173, tag_tid=11, entry_tid=44261' 2026-02-19T20:35:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=173, tag_tid=11, entry_tid=44261' 2026-02-19T20:35:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=173, tag_tid=11, entry_tid=44261' '!=' 'object_number=172, tag_tid=11, entry_tid=43944' 2026-02-19T20:35:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=173, tag_tid=11, entry_tid=44261' = 'object_number=172, tag_tid=11, entry_tid=43944' 2026-02-19T20:35:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=173, tag_tid=11, entry_tid=44261' 2026-02-19T20:35:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=173, tag_tid=11, entry_tid=44261' 2026-02-19T20:35:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:03.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:03.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=173, tag_tid=11, entry_tid=44261' 2026-02-19T20:35:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44261 2026-02-19T20:35:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 44261 2026-02-19T20:35:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:03.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:03.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:03.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:03.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:03.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:03.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=174, tag_tid=11, entry_tid=44558], [object_number=173, tag_tid=11, entry_tid=44557], [object_number=172, tag_tid=11, entry_tid=44556], [object_number=175, tag_tid=11, entry_tid=44555]]], state=connected] 2026-02-19T20:35:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:03.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:03.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:03.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:03.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:03.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=174, tag_tid=11, entry_tid=44558], [object_number=173, tag_tid=11, entry_tid=44557], [object_number=172, tag_tid=11, entry_tid=44556], [object_number=175, tag_tid=11, entry_tid=44555]]], state=connected] 2026-02-19T20:35:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=174, tag_tid=11, entry_tid=44558' 2026-02-19T20:35:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=174, tag_tid=11, entry_tid=44558' 2026-02-19T20:35:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=174, tag_tid=11, entry_tid=44558' '!=' 'object_number=173, tag_tid=11, entry_tid=44261' 2026-02-19T20:35:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=174, tag_tid=11, entry_tid=44558' = 'object_number=173, tag_tid=11, entry_tid=44261' 2026-02-19T20:35:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=174, tag_tid=11, entry_tid=44558' 2026-02-19T20:35:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:03.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:03.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:03.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:03.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=174, tag_tid=11, entry_tid=44558' 2026-02-19T20:35:03.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:03.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:03.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:03.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=174, tag_tid=11, entry_tid=44558' 2026-02-19T20:35:03.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:03.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:03.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44558 2026-02-19T20:35:03.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 44558 2026-02-19T20:35:03.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:03.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:03.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:03.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:03.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:03.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:03.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:03.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:03.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:03.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:03.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:03.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=174, tag_tid=11, entry_tid=44842], [object_number=173, tag_tid=11, entry_tid=44841], [object_number=172, tag_tid=11, entry_tid=44840], [object_number=175, tag_tid=11, entry_tid=44839]]], state=connected] 2026-02-19T20:35:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=174, tag_tid=11, entry_tid=44842], [object_number=173, tag_tid=11, entry_tid=44841], [object_number=172, tag_tid=11, entry_tid=44840], [object_number=175, tag_tid=11, entry_tid=44839]]], state=connected] 2026-02-19T20:35:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=174, tag_tid=11, entry_tid=44842' 2026-02-19T20:35:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=174, tag_tid=11, entry_tid=44842' 2026-02-19T20:35:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=174, tag_tid=11, entry_tid=44842' '!=' 'object_number=174, tag_tid=11, entry_tid=44558' 2026-02-19T20:35:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=174, tag_tid=11, entry_tid=44842' = 'object_number=174, tag_tid=11, entry_tid=44558' 2026-02-19T20:35:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=174, tag_tid=11, entry_tid=44842' 2026-02-19T20:35:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:03.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:03.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:03.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=174, tag_tid=11, entry_tid=44842' 2026-02-19T20:35:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:04.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:04.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:04.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:04.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=174, tag_tid=11, entry_tid=44842' 2026-02-19T20:35:04.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:04.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44842 2026-02-19T20:35:04.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 44842 2026-02-19T20:35:04.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:04.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:04.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:04.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:04.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:04.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:04.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:04.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:04.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:04.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:04.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:04.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:04.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:04.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:04.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:04.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:04.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:04.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:04.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:04.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:04.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=175, tag_tid=11, entry_tid=45123], [object_number=174, tag_tid=11, entry_tid=45122], [object_number=173, tag_tid=11, entry_tid=45121], [object_number=172, tag_tid=11, entry_tid=45120]]], state=connected] 2026-02-19T20:35:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=175, tag_tid=11, entry_tid=45123], [object_number=174, tag_tid=11, entry_tid=45122], [object_number=173, tag_tid=11, entry_tid=45121], [object_number=172, tag_tid=11, entry_tid=45120]]], state=connected] 2026-02-19T20:35:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=175, tag_tid=11, entry_tid=45123' 2026-02-19T20:35:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=175, tag_tid=11, entry_tid=45123' 2026-02-19T20:35:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=175, tag_tid=11, entry_tid=45123' '!=' 'object_number=174, tag_tid=11, entry_tid=44842' 2026-02-19T20:35:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=175, tag_tid=11, entry_tid=45123' = 'object_number=174, tag_tid=11, entry_tid=44842' 2026-02-19T20:35:04.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=175, tag_tid=11, entry_tid=45123' 2026-02-19T20:35:04.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:04.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:04.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:04.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=175, tag_tid=11, entry_tid=45123' 2026-02-19T20:35:04.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:04.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:04.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:04.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:04.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:04.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=175, tag_tid=11, entry_tid=45123' 2026-02-19T20:35:04.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:04.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45123 2026-02-19T20:35:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 45123 2026-02-19T20:35:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:04.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:04.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:04.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:04.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:04.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:04.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:04.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:04.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:04.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:04.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:04.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:04.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:04.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=173, tag_tid=11, entry_tid=45417], [object_number=172, tag_tid=11, entry_tid=45416], [object_number=175, tag_tid=11, entry_tid=45415], [object_number=174, tag_tid=11, entry_tid=45414]]], state=connected] 2026-02-19T20:35:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:04.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:04.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:04.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:04.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=173, tag_tid=11, entry_tid=45417], [object_number=172, tag_tid=11, entry_tid=45416], [object_number=175, tag_tid=11, entry_tid=45415], [object_number=174, tag_tid=11, entry_tid=45414]]], state=connected] 2026-02-19T20:35:04.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=173, tag_tid=11, entry_tid=45417' 2026-02-19T20:35:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=173, tag_tid=11, entry_tid=45417' 2026-02-19T20:35:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=173, tag_tid=11, entry_tid=45417' '!=' 'object_number=175, tag_tid=11, entry_tid=45123' 2026-02-19T20:35:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=173, tag_tid=11, entry_tid=45417' = 'object_number=175, tag_tid=11, entry_tid=45123' 2026-02-19T20:35:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=173, tag_tid=11, entry_tid=45417' 2026-02-19T20:35:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:04.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=173, tag_tid=11, entry_tid=45417' 2026-02-19T20:35:04.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:04.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:04.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:04.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:04.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=173, tag_tid=11, entry_tid=45417' 2026-02-19T20:35:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45417 2026-02-19T20:35:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 45417 2026-02-19T20:35:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:05.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:05.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:05.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:05.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:05.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:05.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:05.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=172, tag_tid=11, entry_tid=45704], [object_number=175, tag_tid=11, entry_tid=45703], [object_number=174, tag_tid=11, entry_tid=45702], [object_number=173, tag_tid=11, entry_tid=45701]]], state=connected] 2026-02-19T20:35:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:05.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:05.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:05.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:05.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:05.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:05.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:05.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:05.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:05.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=172, tag_tid=11, entry_tid=45704], [object_number=175, tag_tid=11, entry_tid=45703], [object_number=174, tag_tid=11, entry_tid=45702], [object_number=173, tag_tid=11, entry_tid=45701]]], state=connected] 2026-02-19T20:35:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=172, tag_tid=11, entry_tid=45704' 2026-02-19T20:35:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=172, tag_tid=11, entry_tid=45704' 2026-02-19T20:35:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=172, tag_tid=11, entry_tid=45704' '!=' 'object_number=173, tag_tid=11, entry_tid=45417' 2026-02-19T20:35:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=172, tag_tid=11, entry_tid=45704' = 'object_number=173, tag_tid=11, entry_tid=45417' 2026-02-19T20:35:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=172, tag_tid=11, entry_tid=45704' 2026-02-19T20:35:05.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:05.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:05.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:05.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:05.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=172, tag_tid=11, entry_tid=45704' 2026-02-19T20:35:05.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:05.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=172, tag_tid=11, entry_tid=45704' 2026-02-19T20:35:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:05.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45704 2026-02-19T20:35:05.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 45704 2026-02-19T20:35:05.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:05.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:05.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:05.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:05.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:05.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:05.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:05.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:05.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:05.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:05.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:05.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:05.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:05.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:05.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:05.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:05.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:05.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=174, tag_tid=11, entry_tid=46126], [object_number=173, tag_tid=11, entry_tid=46125], [object_number=172, tag_tid=11, entry_tid=46124], [object_number=175, tag_tid=11, entry_tid=46123]]], state=connected] 2026-02-19T20:35:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=174, tag_tid=11, entry_tid=46126], [object_number=173, tag_tid=11, entry_tid=46125], [object_number=172, tag_tid=11, entry_tid=46124], [object_number=175, tag_tid=11, entry_tid=46123]]], state=connected] 2026-02-19T20:35:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=174, tag_tid=11, entry_tid=46126' 2026-02-19T20:35:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=174, tag_tid=11, entry_tid=46126' 2026-02-19T20:35:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=174, tag_tid=11, entry_tid=46126' '!=' 'object_number=172, tag_tid=11, entry_tid=45704' 2026-02-19T20:35:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=174, tag_tid=11, entry_tid=46126' = 'object_number=172, tag_tid=11, entry_tid=45704' 2026-02-19T20:35:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=174, tag_tid=11, entry_tid=46126' 2026-02-19T20:35:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=174, tag_tid=11, entry_tid=46126' 2026-02-19T20:35:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:05.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=174, tag_tid=11, entry_tid=46126' 2026-02-19T20:35:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:05.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46126 2026-02-19T20:35:05.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 46126 2026-02-19T20:35:05.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:05.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:05.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:06.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=175, tag_tid=11, entry_tid=46419], [object_number=174, tag_tid=11, entry_tid=46418], [object_number=173, tag_tid=11, entry_tid=46417], [object_number=172, tag_tid=11, entry_tid=46416]]], state=connected] 2026-02-19T20:35:06.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:06.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=175, tag_tid=11, entry_tid=46419], [object_number=174, tag_tid=11, entry_tid=46418], [object_number=173, tag_tid=11, entry_tid=46417], [object_number=172, tag_tid=11, entry_tid=46416]]], state=connected] 2026-02-19T20:35:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=175, tag_tid=11, entry_tid=46419' 2026-02-19T20:35:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=175, tag_tid=11, entry_tid=46419' 2026-02-19T20:35:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=175, tag_tid=11, entry_tid=46419' '!=' 'object_number=174, tag_tid=11, entry_tid=46126' 2026-02-19T20:35:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=175, tag_tid=11, entry_tid=46419' = 'object_number=174, tag_tid=11, entry_tid=46126' 2026-02-19T20:35:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=175, tag_tid=11, entry_tid=46419' 2026-02-19T20:35:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=175, tag_tid=11, entry_tid=46419' 2026-02-19T20:35:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:06.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:06.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:06.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:06.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:06.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=175, tag_tid=11, entry_tid=46419' 2026-02-19T20:35:06.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:06.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:06.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46419 2026-02-19T20:35:06.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 46419 2026-02-19T20:35:06.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:06.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:06.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:06.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:06.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:06.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:06.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:06.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:06.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:06.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:06.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:06.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=174, tag_tid=11, entry_tid=46706], [object_number=173, tag_tid=11, entry_tid=46705], [object_number=172, tag_tid=11, entry_tid=46704], [object_number=175, tag_tid=11, entry_tid=46703]]], state=connected] 2026-02-19T20:35:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=174, tag_tid=11, entry_tid=46706], [object_number=173, tag_tid=11, entry_tid=46705], [object_number=172, tag_tid=11, entry_tid=46704], [object_number=175, tag_tid=11, entry_tid=46703]]], state=connected] 2026-02-19T20:35:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=174, tag_tid=11, entry_tid=46706' 2026-02-19T20:35:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=174, tag_tid=11, entry_tid=46706' 2026-02-19T20:35:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=174, tag_tid=11, entry_tid=46706' '!=' 'object_number=175, tag_tid=11, entry_tid=46419' 2026-02-19T20:35:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=174, tag_tid=11, entry_tid=46706' = 'object_number=175, tag_tid=11, entry_tid=46419' 2026-02-19T20:35:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=174, tag_tid=11, entry_tid=46706' 2026-02-19T20:35:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=174, tag_tid=11, entry_tid=46706' 2026-02-19T20:35:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=174, tag_tid=11, entry_tid=46706' 2026-02-19T20:35:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46706 2026-02-19T20:35:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 46706 2026-02-19T20:35:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:06.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:06.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:06.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:06.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:06.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:06.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:06.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:06.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:06.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:06.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:06.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:06.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:06.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:07.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=175, tag_tid=11, entry_tid=46987], [object_number=174, tag_tid=11, entry_tid=46986], [object_number=173, tag_tid=11, entry_tid=46985], [object_number=172, tag_tid=11, entry_tid=46984]]], state=connected] 2026-02-19T20:35:07.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 43 2026-02-19T20:35:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=175, tag_tid=11, entry_tid=46987], [object_number=174, tag_tid=11, entry_tid=46986], [object_number=173, tag_tid=11, entry_tid=46985], [object_number=172, tag_tid=11, entry_tid=46984]]], state=connected] 2026-02-19T20:35:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=175, tag_tid=11, entry_tid=46987' 2026-02-19T20:35:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=175, tag_tid=11, entry_tid=46987' 2026-02-19T20:35:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=175, tag_tid=11, entry_tid=46987' '!=' 'object_number=174, tag_tid=11, entry_tid=46706' 2026-02-19T20:35:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=175, tag_tid=11, entry_tid=46987' = 'object_number=174, tag_tid=11, entry_tid=46706' 2026-02-19T20:35:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=175, tag_tid=11, entry_tid=46987' 2026-02-19T20:35:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=175, tag_tid=11, entry_tid=46987' 2026-02-19T20:35:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:07.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:07.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:07.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:07.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=175, tag_tid=11, entry_tid=46987' 2026-02-19T20:35:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46987 2026-02-19T20:35:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 46987 2026-02-19T20:35:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:07.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:07.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:07.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:07.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:07.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:07.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:07.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:07.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:07.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:07.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=47221], [object_number=176, tag_tid=11, entry_tid=47220], [object_number=179, tag_tid=11, entry_tid=47219], [object_number=178, tag_tid=11, entry_tid=47218]]], state=connected] 2026-02-19T20:35:07.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:07.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:07.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:07.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:07.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:07.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:07.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=47221], [object_number=176, tag_tid=11, entry_tid=47220], [object_number=179, tag_tid=11, entry_tid=47219], [object_number=178, tag_tid=11, entry_tid=47218]]], state=connected] 2026-02-19T20:35:07.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=177, tag_tid=11, entry_tid=47221' 2026-02-19T20:35:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=177, tag_tid=11, entry_tid=47221' 2026-02-19T20:35:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=47221' '!=' 'object_number=175, tag_tid=11, entry_tid=46987' 2026-02-19T20:35:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=47221' = 'object_number=175, tag_tid=11, entry_tid=46987' 2026-02-19T20:35:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=177, tag_tid=11, entry_tid=47221' 2026-02-19T20:35:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:07.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:07.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=47221' 2026-02-19T20:35:07.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:07.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=47221' 2026-02-19T20:35:07.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:07.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47221 2026-02-19T20:35:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 47221 2026-02-19T20:35:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:07.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:07.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:07.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:07.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:07.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:07.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:07.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:07.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=47501], [object_number=176, tag_tid=11, entry_tid=47500], [object_number=179, tag_tid=11, entry_tid=47499], [object_number=178, tag_tid=11, entry_tid=47498]]], state=connected] 2026-02-19T20:35:08.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=47501], [object_number=176, tag_tid=11, entry_tid=47500], [object_number=179, tag_tid=11, entry_tid=47499], [object_number=178, tag_tid=11, entry_tid=47498]]], state=connected] 2026-02-19T20:35:08.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=177, tag_tid=11, entry_tid=47501' 2026-02-19T20:35:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=177, tag_tid=11, entry_tid=47501' 2026-02-19T20:35:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=47501' '!=' 'object_number=177, tag_tid=11, entry_tid=47221' 2026-02-19T20:35:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=47501' = 'object_number=177, tag_tid=11, entry_tid=47221' 2026-02-19T20:35:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=177, tag_tid=11, entry_tid=47501' 2026-02-19T20:35:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=47501' 2026-02-19T20:35:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=47501' 2026-02-19T20:35:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47501 2026-02-19T20:35:08.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 47501 2026-02-19T20:35:08.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:08.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:08.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:08.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:08.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:08.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:08.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:08.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:08.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:08.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=179, tag_tid=11, entry_tid=47843], [object_number=178, tag_tid=11, entry_tid=47842], [object_number=177, tag_tid=11, entry_tid=47841], [object_number=176, tag_tid=11, entry_tid=47840]]], state=connected] 2026-02-19T20:35:08.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:08.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:08.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=179, tag_tid=11, entry_tid=47843], [object_number=178, tag_tid=11, entry_tid=47842], [object_number=177, tag_tid=11, entry_tid=47841], [object_number=176, tag_tid=11, entry_tid=47840]]], state=connected] 2026-02-19T20:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=179, tag_tid=11, entry_tid=47843' 2026-02-19T20:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=179, tag_tid=11, entry_tid=47843' 2026-02-19T20:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=179, tag_tid=11, entry_tid=47843' '!=' 'object_number=177, tag_tid=11, entry_tid=47501' 2026-02-19T20:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=179, tag_tid=11, entry_tid=47843' = 'object_number=177, tag_tid=11, entry_tid=47501' 2026-02-19T20:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=179, tag_tid=11, entry_tid=47843' 2026-02-19T20:35:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=179, tag_tid=11, entry_tid=47843' 2026-02-19T20:35:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=179, tag_tid=11, entry_tid=47843' 2026-02-19T20:35:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47843 2026-02-19T20:35:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 47843 2026-02-19T20:35:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:08.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:08.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:08.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:08.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:08.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:08.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:08.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:08.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:08.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:08.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:08.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:08.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:08.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:08.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:08.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:08.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:08.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:08.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:08.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:08.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=176, tag_tid=11, entry_tid=48116], [object_number=179, tag_tid=11, entry_tid=48115], [object_number=178, tag_tid=11, entry_tid=48114], [object_number=177, tag_tid=11, entry_tid=48113]]], state=connected] 2026-02-19T20:35:08.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:08.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:08.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:08.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:08.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:08.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:08.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:08.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:08.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:08.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:08.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:08.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:08.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:08.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:08.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:08.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:08.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:08.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:08.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=176, tag_tid=11, entry_tid=48116], [object_number=179, tag_tid=11, entry_tid=48115], [object_number=178, tag_tid=11, entry_tid=48114], [object_number=177, tag_tid=11, entry_tid=48113]]], state=connected] 2026-02-19T20:35:08.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:08.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=176, tag_tid=11, entry_tid=48116' 2026-02-19T20:35:08.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=176, tag_tid=11, entry_tid=48116' 2026-02-19T20:35:08.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=176, tag_tid=11, entry_tid=48116' '!=' 'object_number=179, tag_tid=11, entry_tid=47843' 2026-02-19T20:35:08.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:08.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=176, tag_tid=11, entry_tid=48116' = 'object_number=179, tag_tid=11, entry_tid=47843' 2026-02-19T20:35:08.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=176, tag_tid=11, entry_tid=48116' 2026-02-19T20:35:08.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:08.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:08.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=176, tag_tid=11, entry_tid=48116' 2026-02-19T20:35:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:08.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:08.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:08.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=176, tag_tid=11, entry_tid=48116' 2026-02-19T20:35:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48116 2026-02-19T20:35:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 48116 2026-02-19T20:35:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:09.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:09.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:09.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:09.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:09.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:09.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:09.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:09.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:09.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:09.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:09.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:09.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:09.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:09.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:09.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:09.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:09.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:09.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:09.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=48390], [object_number=177, tag_tid=11, entry_tid=48389], [object_number=176, tag_tid=11, entry_tid=48388], [object_number=179, tag_tid=11, entry_tid=48387]]], state=connected] 2026-02-19T20:35:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:09.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=48390], [object_number=177, tag_tid=11, entry_tid=48389], [object_number=176, tag_tid=11, entry_tid=48388], [object_number=179, tag_tid=11, entry_tid=48387]]], state=connected] 2026-02-19T20:35:09.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=178, tag_tid=11, entry_tid=48390' 2026-02-19T20:35:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=178, tag_tid=11, entry_tid=48390' 2026-02-19T20:35:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=48390' '!=' 'object_number=176, tag_tid=11, entry_tid=48116' 2026-02-19T20:35:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=48390' = 'object_number=176, tag_tid=11, entry_tid=48116' 2026-02-19T20:35:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=178, tag_tid=11, entry_tid=48390' 2026-02-19T20:35:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=48390' 2026-02-19T20:35:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:09.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:09.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:09.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:09.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=48390' 2026-02-19T20:35:09.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:09.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48390 2026-02-19T20:35:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 48390 2026-02-19T20:35:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:09.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:09.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:09.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=48729], [object_number=176, tag_tid=11, entry_tid=48728], [object_number=179, tag_tid=11, entry_tid=48727], [object_number=178, tag_tid=11, entry_tid=48726]]], state=connected] 2026-02-19T20:35:09.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=48729], [object_number=176, tag_tid=11, entry_tid=48728], [object_number=179, tag_tid=11, entry_tid=48727], [object_number=178, tag_tid=11, entry_tid=48726]]], state=connected] 2026-02-19T20:35:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=177, tag_tid=11, entry_tid=48729' 2026-02-19T20:35:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=177, tag_tid=11, entry_tid=48729' 2026-02-19T20:35:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=48729' '!=' 'object_number=178, tag_tid=11, entry_tid=48390' 2026-02-19T20:35:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=48729' = 'object_number=178, tag_tid=11, entry_tid=48390' 2026-02-19T20:35:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=177, tag_tid=11, entry_tid=48729' 2026-02-19T20:35:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:09.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:09.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=48729' 2026-02-19T20:35:09.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:09.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:09.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:09.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:09.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:09.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=48729' 2026-02-19T20:35:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:09.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48729 2026-02-19T20:35:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 48729 2026-02-19T20:35:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:10.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:10.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:10.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:10.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:10.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:10.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:10.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:10.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:10.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:10.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=49001], [object_number=176, tag_tid=11, entry_tid=49000], [object_number=179, tag_tid=11, entry_tid=48999], [object_number=178, tag_tid=11, entry_tid=48998]]], state=connected] 2026-02-19T20:35:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:10.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:10.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:10.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=177, tag_tid=11, entry_tid=49001], [object_number=176, tag_tid=11, entry_tid=49000], [object_number=179, tag_tid=11, entry_tid=48999], [object_number=178, tag_tid=11, entry_tid=48998]]], state=connected] 2026-02-19T20:35:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=177, tag_tid=11, entry_tid=49001' 2026-02-19T20:35:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=177, tag_tid=11, entry_tid=49001' 2026-02-19T20:35:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=49001' '!=' 'object_number=177, tag_tid=11, entry_tid=48729' 2026-02-19T20:35:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=177, tag_tid=11, entry_tid=49001' = 'object_number=177, tag_tid=11, entry_tid=48729' 2026-02-19T20:35:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=177, tag_tid=11, entry_tid=49001' 2026-02-19T20:35:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=49001' 2026-02-19T20:35:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=11, entry_tid=49001' 2026-02-19T20:35:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=49001 2026-02-19T20:35:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 49001 2026-02-19T20:35:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:10.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:10.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:10.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:10.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:10.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:10.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:10.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:10.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:10.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:10.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:10.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:10.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:10.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:10.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:10.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:10.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:10.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:10.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=49298], [object_number=177, tag_tid=11, entry_tid=49297], [object_number=176, tag_tid=11, entry_tid=49296], [object_number=179, tag_tid=11, entry_tid=49295]]], state=connected] 2026-02-19T20:35:10.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:10.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:10.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:10.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:10.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=178, tag_tid=11, entry_tid=49298], [object_number=177, tag_tid=11, entry_tid=49297], [object_number=176, tag_tid=11, entry_tid=49296], [object_number=179, tag_tid=11, entry_tid=49295]]], state=connected] 2026-02-19T20:35:10.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=178, tag_tid=11, entry_tid=49298' 2026-02-19T20:35:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=178, tag_tid=11, entry_tid=49298' 2026-02-19T20:35:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=49298' '!=' 'object_number=177, tag_tid=11, entry_tid=49001' 2026-02-19T20:35:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:35:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=178, tag_tid=11, entry_tid=49298' = 'object_number=177, tag_tid=11, entry_tid=49001' 2026-02-19T20:35:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=178, tag_tid=11, entry_tid=49298' 2026-02-19T20:35:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:10.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=12 2026-02-19T20:35:10.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=49298' 2026-02-19T20:35:10.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:35:10.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:10.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=11 2026-02-19T20:35:10.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:10.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:10.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:10.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:35:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=178, tag_tid=11, entry_tid=49298' 2026-02-19T20:35:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:35:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:35:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=49298 2026-02-19T20:35:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 12 = 11 -a 1 -le 49298 2026-02-19T20:35:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:35:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:35:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:35:10.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:35:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:35:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:35:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:35:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:35:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:35:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:35:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:35:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:10.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:35:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:35:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:10.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:10.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:35:10.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:35:10.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:35:10.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:35:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:35:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:35:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:35:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:35:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:35:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:35:10.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:10.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:10.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:10.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:10.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:35:10.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:35:10.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:10.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:10.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:10.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:10.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:35:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:35:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:35:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:35:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:11.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:11.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:35:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:35:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:35:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:35:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:35:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:35:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:35:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:35:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:35:11.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:11.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:11.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:35:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 44 2026-02-19T20:35:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 44 2026-02-19T20:35:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:35:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=177, tag_tid=12, entry_tid=1], [object_number=176, tag_tid=12, entry_tid=0], [object_number=179, tag_tid=11, entry_tid=49319], [object_number=178, tag_tid=11, entry_tid=49318]]], state=connected] 2026-02-19T20:35:11.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:35:11.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:11.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=177, tag_tid=12, entry_tid=1' -a 'object_number=177, tag_tid=12, entry_tid=1' = 'object_number=177, tag_tid=12, entry_tid=1' 2026-02-19T20:35:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2026-02-19T20:35:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap4 2026-02-19T20:35:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:35:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:35:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:35:12.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2026-02-19T20:35:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:35:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:35:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:35:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 7 2026-02-19T20:35:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:35:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:35:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:35:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:33:53 2026 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:34:24 2026 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:35:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:35:12.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:12.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n snap3-clean ']' 2026-02-19T20:35:12.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-19T20:35:12.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:12.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:12.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap3-clean 2026-02-19T20:35:12.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:35:12.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:35:12.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:35:12.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:35:12.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:35:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:35:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:35:13.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:35:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:35:13.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:35:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap4 2026-02-19T20:35:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap4 2026-02-19T20:35:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:35:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:35:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:35:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:35:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:35:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:35:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:35:14.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:35:14.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:35:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:35:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name=snap4-clean 2026-02-19T20:35:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap4-clean 2026-02-19T20:35:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:35:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:15.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:15.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap4-clean 2026-02-19T20:35:15.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2026-02-19T20:35:15.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:35:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror test 2026-02-19T20:35:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:35:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:35:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=10 2026-02-19T20:35:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:35:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-19T20:35:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 39480320 bytes 2026-02-19T20:35:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 82971648 bytes 2026-02-19T20:35:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 139411456 bytes 2026-02-19T20:35:18.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 208112640 bytes 2026-02-19T20:35:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 230382592 bytes 2026-02-19T20:35:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 293619712 bytes 2026-02-19T20:35:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 373810176 bytes 2026-02-19T20:35:21.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 411335680 bytes 2026-02-19T20:35:22.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 442390528 bytes 2026-02-19T20:35:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 471678976 bytes 2026-02-19T20:35:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 562727936 bytes 2026-02-19T20:35:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 585358336 bytes 2026-02-19T20:35:24.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 598331392 bytes 2026-02-19T20:35:24.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 645975040 bytes 2026-02-19T20:35:25.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 723338240 bytes 2026-02-19T20:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-19T20:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:35:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-19T20:35:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:35:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:35:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:35:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:35:26.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:35:26.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-19T20:35:26.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='test: 2026-02-19T20:35:26.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: global_id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:35:26.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+replaying 2026-02-19T20:35:26.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: replaying, {"bytes_per_second":10672092.4,"entries_behind_primary":0,"entries_per_second":671.0,"non_primary_position":{"entry_tid":1,"object_number":177,"tag_tid":12},"primary_position":{"entry_tid":1,"object_number":177,"tag_tid":12}} 2026-02-19T20:35:26.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: service: mirror.1 on trial151 2026-02-19T20:35:26.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:35:17 2026-02-19T20:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: peer_sites: 2026-02-19T20:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: name: cluster2 2026-02-19T20:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+stopped 2026-02-19T20:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: local image is primary 2026-02-19T20:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:35:18' 2026-02-19T20:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ state: .*up+replaying' 2026-02-19T20:35:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: state: up+replaying 2026-02-19T20:35:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ description: .*' 2026-02-19T20:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: description: replaying, {"bytes_per_second":10672092.4,"entries_behind_primary":0,"entries_per_second":671.0,"non_primary_position":{"entry_tid":1,"object_number":177,"tag_tid":12},"primary_position":{"entry_tid":1,"object_number":177,"tag_tid":12}} 2026-02-19T20:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:35:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^up+' 2026-02-19T20:35:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:35:26.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep 'service: *mirror..* on ' 2026-02-19T20:35:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: service: mirror.1 on trial151 2026-02-19T20:35:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update 2026-02-19T20:35:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-19T20:35:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:35:17' 2026-02-19T20:35:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-19 20:35:17' 2026-02-19T20:35:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:35:26.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'prev_last_update=2026-02-19 20:35:17' 2026-02-19T20:35:26.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:35:26.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:35:26.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-19T20:35:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='OKOKOK142064262mirror.1trial15120.3.0-5336-g802bf111falseOK42104271mirror.2trial15120.3.0-5336-g802bf111trueOK43494352mirror.0trial15120.3.0-5336-g802bf111falseOK43714374mirror.3trial15120.3.0-5336-g802bf111falseOKtest959deb13-de1a-48d9-a341-1f95416f8379up+replayingreplaying, {"bytes_per_second":10672092.4,"entries_behind_primary":0,"entries_per_second":671.0,"non_primary_position":{"entry_tid":1,"object_number":177,"tag_tid":12},"primary_position":{"entry_tid":1,"object_number":177,"tag_tid":12}}42064262mirror.1trial1512026-02-19 20:35:17cluster2a054b997-3cfb-48e6-978a-70d6a85e56deup+stoppedlocal image is primary2026-02-19 20:35:18' 2026-02-19T20:35:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update state 2026-02-19T20:35:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-19T20:35:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:35:17' 2026-02-19T20:35:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-19T20:35:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=up+replaying 2026-02-19T20:35:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:35:26.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep up+replaying 2026-02-19T20:35:26.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:35:26.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:26.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:26.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap5 2026-02-19T20:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap5 2026-02-19T20:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap5 2026-02-19T20:35:26.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:35:58.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:58.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:35:59.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:35:59.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:35:59.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:35:59.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:35:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:35:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:35:59.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:35:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:35:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:35:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:35:59.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:59.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:35:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:35:59.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:35:59.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:35:59.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:35:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:35:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:35:59.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:59.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:59.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:35:59.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:35:59.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:35:59.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:35:59.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:35:59.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:35:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:35:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:35:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:35:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:35:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:35:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:35:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:35:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:35:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:35:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:35:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:00.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:00.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:00.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:00.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:00.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:00.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:00.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:00.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:00.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:00.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=28339], [object_number=206, tag_tid=14, entry_tid=28338], [object_number=205, tag_tid=14, entry_tid=28337], [object_number=204, tag_tid=14, entry_tid=28336]]], state=connected] 2026-02-19T20:36:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=28339], [object_number=206, tag_tid=14, entry_tid=28338], [object_number=205, tag_tid=14, entry_tid=28337], [object_number=204, tag_tid=14, entry_tid=28336]]], state=connected] 2026-02-19T20:36:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=207, tag_tid=14, entry_tid=28339' 2026-02-19T20:36:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=207, tag_tid=14, entry_tid=28339' 2026-02-19T20:36:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=28339' '!=' '' 2026-02-19T20:36:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=28339' = '' 2026-02-19T20:36:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=207, tag_tid=14, entry_tid=28339' 2026-02-19T20:36:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:00.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:00.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=28339' 2026-02-19T20:36:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:00.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=28339' 2026-02-19T20:36:00.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:00.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28339 2026-02-19T20:36:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 28339 2026-02-19T20:36:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:00.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:00.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:00.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:00.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:00.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:00.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:00.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:00.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:00.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=28631], [object_number=206, tag_tid=14, entry_tid=28630], [object_number=205, tag_tid=14, entry_tid=28629], [object_number=204, tag_tid=14, entry_tid=28628]]], state=connected] 2026-02-19T20:36:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:00.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:00.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:00.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:00.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=28631], [object_number=206, tag_tid=14, entry_tid=28630], [object_number=205, tag_tid=14, entry_tid=28629], [object_number=204, tag_tid=14, entry_tid=28628]]], state=connected] 2026-02-19T20:36:00.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:00.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=207, tag_tid=14, entry_tid=28631' 2026-02-19T20:36:00.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=207, tag_tid=14, entry_tid=28631' 2026-02-19T20:36:00.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=28631' '!=' 'object_number=207, tag_tid=14, entry_tid=28339' 2026-02-19T20:36:00.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=28631' = 'object_number=207, tag_tid=14, entry_tid=28339' 2026-02-19T20:36:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=207, tag_tid=14, entry_tid=28631' 2026-02-19T20:36:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=28631' 2026-02-19T20:36:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:00.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:00.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:00.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:00.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:00.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=28631' 2026-02-19T20:36:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28631 2026-02-19T20:36:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 28631 2026-02-19T20:36:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:00.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:00.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:00.785 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:36:00.779+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:36:00.785 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:36:00.779+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:36:00.785 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:36:00.779+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:36:00.785 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:36:00.779+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00d096c0 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:00.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:00.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:00.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:00.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:01.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:01.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:01.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:01.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:01.043 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T20:36:01.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T20:36:01.043 INFO:tasks.rbd_mirror_thrash:waiting for 6.1886089749324285 secs before reviving daemons 2026-02-19T20:36:01.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T20:36:01.037+0000 7fb0c51f1640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 32772) UID: 1000 2026-02-19T20:36:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:01.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:01.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:01.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:01.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=28924], [object_number=207, tag_tid=14, entry_tid=28923], [object_number=206, tag_tid=14, entry_tid=28922], [object_number=205, tag_tid=14, entry_tid=28921]]], state=connected] 2026-02-19T20:36:01.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:01.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:01.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:01.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:01.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:01.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=28924], [object_number=207, tag_tid=14, entry_tid=28923], [object_number=206, tag_tid=14, entry_tid=28922], [object_number=205, tag_tid=14, entry_tid=28921]]], state=connected] 2026-02-19T20:36:01.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=204, tag_tid=14, entry_tid=28924' 2026-02-19T20:36:01.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=204, tag_tid=14, entry_tid=28924' 2026-02-19T20:36:01.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=28924' '!=' 'object_number=207, tag_tid=14, entry_tid=28631' 2026-02-19T20:36:01.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:01.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=28924' = 'object_number=207, tag_tid=14, entry_tid=28631' 2026-02-19T20:36:01.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=204, tag_tid=14, entry_tid=28924' 2026-02-19T20:36:01.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:01.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:01.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:01.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=28924' 2026-02-19T20:36:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:01.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:01.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:01.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:01.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:01.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:01.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=28924' 2026-02-19T20:36:01.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:01.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28924 2026-02-19T20:36:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 28924 2026-02-19T20:36:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:01.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:01.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:01.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:01.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:01.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:01.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:01.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:01.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:01.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:01.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:01.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:01.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:01.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:01.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:01.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:01.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:01.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:01.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:01.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:01.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:01.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:01.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:01.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=29214], [object_number=205, tag_tid=14, entry_tid=29213], [object_number=204, tag_tid=14, entry_tid=29212], [object_number=207, tag_tid=14, entry_tid=29211]]], state=connected] 2026-02-19T20:36:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=29214], [object_number=205, tag_tid=14, entry_tid=29213], [object_number=204, tag_tid=14, entry_tid=29212], [object_number=207, tag_tid=14, entry_tid=29211]]], state=connected] 2026-02-19T20:36:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=206, tag_tid=14, entry_tid=29214' 2026-02-19T20:36:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=206, tag_tid=14, entry_tid=29214' 2026-02-19T20:36:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=29214' '!=' 'object_number=204, tag_tid=14, entry_tid=28924' 2026-02-19T20:36:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=29214' = 'object_number=204, tag_tid=14, entry_tid=28924' 2026-02-19T20:36:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=206, tag_tid=14, entry_tid=29214' 2026-02-19T20:36:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=29214' 2026-02-19T20:36:01.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:01.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:01.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=29214' 2026-02-19T20:36:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29214 2026-02-19T20:36:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 29214 2026-02-19T20:36:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:01.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:01.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:01.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:01.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:01.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:01.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:01.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:01.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:01.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:01.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:01.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:01.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=29490], [object_number=205, tag_tid=14, entry_tid=29489], [object_number=204, tag_tid=14, entry_tid=29488], [object_number=207, tag_tid=14, entry_tid=29487]]], state=connected] 2026-02-19T20:36:01.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:01.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=29490], [object_number=205, tag_tid=14, entry_tid=29489], [object_number=204, tag_tid=14, entry_tid=29488], [object_number=207, tag_tid=14, entry_tid=29487]]], state=connected] 2026-02-19T20:36:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=206, tag_tid=14, entry_tid=29490' 2026-02-19T20:36:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=206, tag_tid=14, entry_tid=29490' 2026-02-19T20:36:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=29490' '!=' 'object_number=206, tag_tid=14, entry_tid=29214' 2026-02-19T20:36:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=29490' = 'object_number=206, tag_tid=14, entry_tid=29214' 2026-02-19T20:36:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=206, tag_tid=14, entry_tid=29490' 2026-02-19T20:36:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:02.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:02.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=29490' 2026-02-19T20:36:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=29490' 2026-02-19T20:36:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29490 2026-02-19T20:36:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 29490 2026-02-19T20:36:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:02.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:02.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:02.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=29751], [object_number=206, tag_tid=14, entry_tid=29750], [object_number=205, tag_tid=14, entry_tid=29749], [object_number=204, tag_tid=14, entry_tid=29748]]], state=connected] 2026-02-19T20:36:02.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:02.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:02.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:02.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:02.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=29751], [object_number=206, tag_tid=14, entry_tid=29750], [object_number=205, tag_tid=14, entry_tid=29749], [object_number=204, tag_tid=14, entry_tid=29748]]], state=connected] 2026-02-19T20:36:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=207, tag_tid=14, entry_tid=29751' 2026-02-19T20:36:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=207, tag_tid=14, entry_tid=29751' 2026-02-19T20:36:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=29751' '!=' 'object_number=206, tag_tid=14, entry_tid=29490' 2026-02-19T20:36:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=29751' = 'object_number=206, tag_tid=14, entry_tid=29490' 2026-02-19T20:36:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=207, tag_tid=14, entry_tid=29751' 2026-02-19T20:36:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:02.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=29751' 2026-02-19T20:36:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=29751' 2026-02-19T20:36:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29751 2026-02-19T20:36:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 29751 2026-02-19T20:36:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:02.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:02.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=30024], [object_number=207, tag_tid=14, entry_tid=30023], [object_number=206, tag_tid=14, entry_tid=30022], [object_number=205, tag_tid=14, entry_tid=30021]]], state=connected] 2026-02-19T20:36:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:02.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:02.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:02.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:02.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:02.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:02.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=30024], [object_number=207, tag_tid=14, entry_tid=30023], [object_number=206, tag_tid=14, entry_tid=30022], [object_number=205, tag_tid=14, entry_tid=30021]]], state=connected] 2026-02-19T20:36:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=204, tag_tid=14, entry_tid=30024' 2026-02-19T20:36:03.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=204, tag_tid=14, entry_tid=30024' 2026-02-19T20:36:03.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=30024' '!=' 'object_number=207, tag_tid=14, entry_tid=29751' 2026-02-19T20:36:03.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:03.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=30024' = 'object_number=207, tag_tid=14, entry_tid=29751' 2026-02-19T20:36:03.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=204, tag_tid=14, entry_tid=30024' 2026-02-19T20:36:03.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:03.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:03.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=30024' 2026-02-19T20:36:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:03.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=30024' 2026-02-19T20:36:03.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:03.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30024 2026-02-19T20:36:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 30024 2026-02-19T20:36:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:03.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:03.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:03.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:03.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=30443], [object_number=206, tag_tid=14, entry_tid=30442], [object_number=205, tag_tid=14, entry_tid=30441], [object_number=204, tag_tid=14, entry_tid=30440]]], state=connected] 2026-02-19T20:36:03.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:03.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=30443], [object_number=206, tag_tid=14, entry_tid=30442], [object_number=205, tag_tid=14, entry_tid=30441], [object_number=204, tag_tid=14, entry_tid=30440]]], state=connected] 2026-02-19T20:36:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=207, tag_tid=14, entry_tid=30443' 2026-02-19T20:36:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=207, tag_tid=14, entry_tid=30443' 2026-02-19T20:36:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=30443' '!=' 'object_number=204, tag_tid=14, entry_tid=30024' 2026-02-19T20:36:03.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=30443' = 'object_number=204, tag_tid=14, entry_tid=30024' 2026-02-19T20:36:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=207, tag_tid=14, entry_tid=30443' 2026-02-19T20:36:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=30443' 2026-02-19T20:36:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:03.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=30443' 2026-02-19T20:36:03.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:03.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30443 2026-02-19T20:36:03.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 30443 2026-02-19T20:36:03.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:03.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:03.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:03.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:03.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:03.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:03.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:03.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=30737], [object_number=204, tag_tid=14, entry_tid=30736], [object_number=207, tag_tid=14, entry_tid=30735], [object_number=206, tag_tid=14, entry_tid=30734]]], state=connected] 2026-02-19T20:36:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 51 2026-02-19T20:36:03.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:03.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:03.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:03.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=30737], [object_number=204, tag_tid=14, entry_tid=30736], [object_number=207, tag_tid=14, entry_tid=30735], [object_number=206, tag_tid=14, entry_tid=30734]]], state=connected] 2026-02-19T20:36:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=209, tag_tid=14, entry_tid=30737' 2026-02-19T20:36:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=209, tag_tid=14, entry_tid=30737' 2026-02-19T20:36:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=30737' '!=' 'object_number=207, tag_tid=14, entry_tid=30443' 2026-02-19T20:36:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=30737' = 'object_number=207, tag_tid=14, entry_tid=30443' 2026-02-19T20:36:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=209, tag_tid=14, entry_tid=30737' 2026-02-19T20:36:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:03.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=30737' 2026-02-19T20:36:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:03.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=30737' 2026-02-19T20:36:03.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:03.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30737 2026-02-19T20:36:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 30737 2026-02-19T20:36:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:04.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:04.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:04.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:04.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:04.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:04.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:04.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:04.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:04.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:04.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:04.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:04.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:04.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:04.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:04.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:04.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:04.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:04.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=30941], [object_number=208, tag_tid=14, entry_tid=30940], [object_number=211, tag_tid=14, entry_tid=30939], [object_number=210, tag_tid=14, entry_tid=30938]]], state=connected] 2026-02-19T20:36:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:04.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:04.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:04.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=30941], [object_number=208, tag_tid=14, entry_tid=30940], [object_number=211, tag_tid=14, entry_tid=30939], [object_number=210, tag_tid=14, entry_tid=30938]]], state=connected] 2026-02-19T20:36:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=209, tag_tid=14, entry_tid=30941' 2026-02-19T20:36:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=209, tag_tid=14, entry_tid=30941' 2026-02-19T20:36:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=30941' '!=' 'object_number=209, tag_tid=14, entry_tid=30737' 2026-02-19T20:36:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=30941' = 'object_number=209, tag_tid=14, entry_tid=30737' 2026-02-19T20:36:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=209, tag_tid=14, entry_tid=30941' 2026-02-19T20:36:04.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:04.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:04.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:04.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:04.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=30941' 2026-02-19T20:36:04.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:04.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:04.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=30941' 2026-02-19T20:36:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30941 2026-02-19T20:36:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 30941 2026-02-19T20:36:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:04.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:04.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:04.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:04.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:04.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:04.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:04.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:04.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:04.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:04.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:04.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:04.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:04.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:04.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=31234], [object_number=209, tag_tid=14, entry_tid=31233], [object_number=208, tag_tid=14, entry_tid=31232], [object_number=211, tag_tid=14, entry_tid=31231]]], state=connected] 2026-02-19T20:36:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:04.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:04.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:04.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:04.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:04.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=31234], [object_number=209, tag_tid=14, entry_tid=31233], [object_number=208, tag_tid=14, entry_tid=31232], [object_number=211, tag_tid=14, entry_tid=31231]]], state=connected] 2026-02-19T20:36:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=210, tag_tid=14, entry_tid=31234' 2026-02-19T20:36:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=210, tag_tid=14, entry_tid=31234' 2026-02-19T20:36:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=31234' '!=' 'object_number=209, tag_tid=14, entry_tid=30941' 2026-02-19T20:36:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=31234' = 'object_number=209, tag_tid=14, entry_tid=30941' 2026-02-19T20:36:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=210, tag_tid=14, entry_tid=31234' 2026-02-19T20:36:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=31234' 2026-02-19T20:36:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:04.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:04.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:04.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=31234' 2026-02-19T20:36:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31234 2026-02-19T20:36:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 31234 2026-02-19T20:36:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:04.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:04.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:04.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:04.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:04.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:04.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:04.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:04.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:04.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:05.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:05.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:05.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:05.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:05.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=31548], [object_number=211, tag_tid=14, entry_tid=31547], [object_number=210, tag_tid=14, entry_tid=31546], [object_number=209, tag_tid=14, entry_tid=31545]]], state=connected] 2026-02-19T20:36:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=31548], [object_number=211, tag_tid=14, entry_tid=31547], [object_number=210, tag_tid=14, entry_tid=31546], [object_number=209, tag_tid=14, entry_tid=31545]]], state=connected] 2026-02-19T20:36:05.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=31548' 2026-02-19T20:36:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=31548' 2026-02-19T20:36:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=31548' '!=' 'object_number=210, tag_tid=14, entry_tid=31234' 2026-02-19T20:36:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=31548' = 'object_number=210, tag_tid=14, entry_tid=31234' 2026-02-19T20:36:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=31548' 2026-02-19T20:36:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:05.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=31548' 2026-02-19T20:36:05.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:05.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:05.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:05.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:05.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=31548' 2026-02-19T20:36:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31548 2026-02-19T20:36:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 31548 2026-02-19T20:36:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:05.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:05.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:05.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=31821], [object_number=208, tag_tid=14, entry_tid=31820], [object_number=211, tag_tid=14, entry_tid=31819], [object_number=210, tag_tid=14, entry_tid=31818]]], state=connected] 2026-02-19T20:36:05.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:05.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:05.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:05.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:05.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=31821], [object_number=208, tag_tid=14, entry_tid=31820], [object_number=211, tag_tid=14, entry_tid=31819], [object_number=210, tag_tid=14, entry_tid=31818]]], state=connected] 2026-02-19T20:36:05.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=209, tag_tid=14, entry_tid=31821' 2026-02-19T20:36:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=209, tag_tid=14, entry_tid=31821' 2026-02-19T20:36:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=31821' '!=' 'object_number=208, tag_tid=14, entry_tid=31548' 2026-02-19T20:36:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=31821' = 'object_number=208, tag_tid=14, entry_tid=31548' 2026-02-19T20:36:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=209, tag_tid=14, entry_tid=31821' 2026-02-19T20:36:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=31821' 2026-02-19T20:36:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:05.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:05.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=31821' 2026-02-19T20:36:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31821 2026-02-19T20:36:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 31821 2026-02-19T20:36:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:05.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:05.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:05.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:05.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:05.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:05.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=32082], [object_number=209, tag_tid=14, entry_tid=32081], [object_number=208, tag_tid=14, entry_tid=32080], [object_number=211, tag_tid=14, entry_tid=32079]]], state=connected] 2026-02-19T20:36:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:05.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:05.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:05.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:05.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:06.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:06.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:06.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:06.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=32082], [object_number=209, tag_tid=14, entry_tid=32081], [object_number=208, tag_tid=14, entry_tid=32080], [object_number=211, tag_tid=14, entry_tid=32079]]], state=connected] 2026-02-19T20:36:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=210, tag_tid=14, entry_tid=32082' 2026-02-19T20:36:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=210, tag_tid=14, entry_tid=32082' 2026-02-19T20:36:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=32082' '!=' 'object_number=209, tag_tid=14, entry_tid=31821' 2026-02-19T20:36:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=32082' = 'object_number=209, tag_tid=14, entry_tid=31821' 2026-02-19T20:36:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=210, tag_tid=14, entry_tid=32082' 2026-02-19T20:36:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=32082' 2026-02-19T20:36:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=32082' 2026-02-19T20:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:06.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32082 2026-02-19T20:36:06.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 32082 2026-02-19T20:36:06.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:06.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:06.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:06.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:06.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=32364], [object_number=211, tag_tid=14, entry_tid=32363], [object_number=210, tag_tid=14, entry_tid=32362], [object_number=209, tag_tid=14, entry_tid=32361]]], state=connected] 2026-02-19T20:36:06.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=32364], [object_number=211, tag_tid=14, entry_tid=32363], [object_number=210, tag_tid=14, entry_tid=32362], [object_number=209, tag_tid=14, entry_tid=32361]]], state=connected] 2026-02-19T20:36:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=32364' 2026-02-19T20:36:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=32364' 2026-02-19T20:36:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=32364' '!=' 'object_number=210, tag_tid=14, entry_tid=32082' 2026-02-19T20:36:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=32364' = 'object_number=210, tag_tid=14, entry_tid=32082' 2026-02-19T20:36:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=32364' 2026-02-19T20:36:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:06.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=32364' 2026-02-19T20:36:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:06.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:06.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:06.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:06.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:06.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=32364' 2026-02-19T20:36:06.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:06.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:06.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32364 2026-02-19T20:36:06.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 32364 2026-02-19T20:36:06.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:06.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:06.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:06.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:06.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:06.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:06.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:06.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:06.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:06.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:36:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:06.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:06.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:06.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=32670], [object_number=209, tag_tid=14, entry_tid=32669], [object_number=208, tag_tid=14, entry_tid=32668], [object_number=211, tag_tid=14, entry_tid=32667]]], state=connected] 2026-02-19T20:36:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=32670], [object_number=209, tag_tid=14, entry_tid=32669], [object_number=208, tag_tid=14, entry_tid=32668], [object_number=211, tag_tid=14, entry_tid=32667]]], state=connected] 2026-02-19T20:36:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=210, tag_tid=14, entry_tid=32670' 2026-02-19T20:36:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=210, tag_tid=14, entry_tid=32670' 2026-02-19T20:36:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=32670' '!=' 'object_number=208, tag_tid=14, entry_tid=32364' 2026-02-19T20:36:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=32670' = 'object_number=208, tag_tid=14, entry_tid=32364' 2026-02-19T20:36:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=210, tag_tid=14, entry_tid=32670' 2026-02-19T20:36:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=32670' 2026-02-19T20:36:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:07.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=32670' 2026-02-19T20:36:07.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:07.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32670 2026-02-19T20:36:07.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 32670 2026-02-19T20:36:07.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:07.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:07.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:07.232 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T20:36:07.233 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:36:07.233 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T20:36:07.233 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T20:36:07.233 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T20:36:07.236 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T20:36:07.236 INFO:tasks.rbd_mirror_thrash:waiting for 61.29596753420573 secs before thrashing 2026-02-19T20:36:07.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:07.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:07.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:07.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:07.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:07.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:07.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:07.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:07.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:07.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:07.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:07.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:07.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:07.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:07.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:07.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:07.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:07.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:07.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:07.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:07.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:07.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:07.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:07.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=33091], [object_number=210, tag_tid=14, entry_tid=33090], [object_number=209, tag_tid=14, entry_tid=33089], [object_number=208, tag_tid=14, entry_tid=33088]]], state=connected] 2026-02-19T20:36:07.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:07.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:07.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:07.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:07.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:07.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:07.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:07.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=33091], [object_number=210, tag_tid=14, entry_tid=33090], [object_number=209, tag_tid=14, entry_tid=33089], [object_number=208, tag_tid=14, entry_tid=33088]]], state=connected] 2026-02-19T20:36:07.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=211, tag_tid=14, entry_tid=33091' 2026-02-19T20:36:07.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=211, tag_tid=14, entry_tid=33091' 2026-02-19T20:36:07.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=33091' '!=' 'object_number=210, tag_tid=14, entry_tid=32670' 2026-02-19T20:36:07.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:07.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=33091' = 'object_number=210, tag_tid=14, entry_tid=32670' 2026-02-19T20:36:07.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=211, tag_tid=14, entry_tid=33091' 2026-02-19T20:36:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:07.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=33091' 2026-02-19T20:36:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:07.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=33091' 2026-02-19T20:36:07.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:07.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33091 2026-02-19T20:36:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 33091 2026-02-19T20:36:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:07.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:07.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=33367], [object_number=210, tag_tid=14, entry_tid=33366], [object_number=209, tag_tid=14, entry_tid=33365], [object_number=208, tag_tid=14, entry_tid=33364]]], state=connected] 2026-02-19T20:36:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:07.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=33367], [object_number=210, tag_tid=14, entry_tid=33366], [object_number=209, tag_tid=14, entry_tid=33365], [object_number=208, tag_tid=14, entry_tid=33364]]], state=connected] 2026-02-19T20:36:07.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:07.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=211, tag_tid=14, entry_tid=33367' 2026-02-19T20:36:07.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=211, tag_tid=14, entry_tid=33367' 2026-02-19T20:36:07.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=33367' '!=' 'object_number=211, tag_tid=14, entry_tid=33091' 2026-02-19T20:36:07.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:07.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=33367' = 'object_number=211, tag_tid=14, entry_tid=33091' 2026-02-19T20:36:07.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=211, tag_tid=14, entry_tid=33367' 2026-02-19T20:36:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=33367' 2026-02-19T20:36:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=33367' 2026-02-19T20:36:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33367 2026-02-19T20:36:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 33367 2026-02-19T20:36:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=33628], [object_number=211, tag_tid=14, entry_tid=33627], [object_number=210, tag_tid=14, entry_tid=33626], [object_number=209, tag_tid=14, entry_tid=33625]]], state=connected] 2026-02-19T20:36:08.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:08.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=33628], [object_number=211, tag_tid=14, entry_tid=33627], [object_number=210, tag_tid=14, entry_tid=33626], [object_number=209, tag_tid=14, entry_tid=33625]]], state=connected] 2026-02-19T20:36:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=33628' 2026-02-19T20:36:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=33628' 2026-02-19T20:36:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=33628' '!=' 'object_number=211, tag_tid=14, entry_tid=33367' 2026-02-19T20:36:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=33628' = 'object_number=211, tag_tid=14, entry_tid=33367' 2026-02-19T20:36:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=33628' 2026-02-19T20:36:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:08.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:08.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:08.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:08.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=33628' 2026-02-19T20:36:08.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:08.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:08.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:08.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:08.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=33628' 2026-02-19T20:36:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33628 2026-02-19T20:36:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 33628 2026-02-19T20:36:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:08.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:08.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:08.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:08.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:08.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:08.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:08.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:08.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:08.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:08.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:08.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=33887], [object_number=210, tag_tid=14, entry_tid=33886], [object_number=209, tag_tid=14, entry_tid=33885], [object_number=208, tag_tid=14, entry_tid=33884]]], state=connected] 2026-02-19T20:36:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:08.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:08.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=33887], [object_number=210, tag_tid=14, entry_tid=33886], [object_number=209, tag_tid=14, entry_tid=33885], [object_number=208, tag_tid=14, entry_tid=33884]]], state=connected] 2026-02-19T20:36:08.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=211, tag_tid=14, entry_tid=33887' 2026-02-19T20:36:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=211, tag_tid=14, entry_tid=33887' 2026-02-19T20:36:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=33887' '!=' 'object_number=208, tag_tid=14, entry_tid=33628' 2026-02-19T20:36:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=33887' = 'object_number=208, tag_tid=14, entry_tid=33628' 2026-02-19T20:36:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=211, tag_tid=14, entry_tid=33887' 2026-02-19T20:36:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=33887' 2026-02-19T20:36:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:08.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:08.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=33887' 2026-02-19T20:36:08.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:08.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:08.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33887 2026-02-19T20:36:08.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 33887 2026-02-19T20:36:08.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:08.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:08.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:09.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=34161], [object_number=208, tag_tid=14, entry_tid=34160], [object_number=211, tag_tid=14, entry_tid=34159], [object_number=210, tag_tid=14, entry_tid=34158]]], state=connected] 2026-02-19T20:36:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:09.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:09.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:09.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=34161], [object_number=208, tag_tid=14, entry_tid=34160], [object_number=211, tag_tid=14, entry_tid=34159], [object_number=210, tag_tid=14, entry_tid=34158]]], state=connected] 2026-02-19T20:36:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=209, tag_tid=14, entry_tid=34161' 2026-02-19T20:36:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=209, tag_tid=14, entry_tid=34161' 2026-02-19T20:36:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=34161' '!=' 'object_number=211, tag_tid=14, entry_tid=33887' 2026-02-19T20:36:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=34161' = 'object_number=211, tag_tid=14, entry_tid=33887' 2026-02-19T20:36:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=209, tag_tid=14, entry_tid=34161' 2026-02-19T20:36:09.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:09.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:09.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:09.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:09.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=34161' 2026-02-19T20:36:09.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:09.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:09.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:09.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:09.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=34161' 2026-02-19T20:36:09.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:09.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34161 2026-02-19T20:36:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 34161 2026-02-19T20:36:09.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:09.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:09.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:09.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:09.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:09.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:09.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:09.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:09.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:09.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:09.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:09.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:09.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:09.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:09.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:09.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:09.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:09.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:09.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:09.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:09.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:09.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:09.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:09.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:09.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:09.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:09.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:09.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:09.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:09.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:09.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:09.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:09.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:09.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:09.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:09.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:09.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:09.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=34442], [object_number=209, tag_tid=14, entry_tid=34441], [object_number=208, tag_tid=14, entry_tid=34440], [object_number=211, tag_tid=14, entry_tid=34439]]], state=connected] 2026-02-19T20:36:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:09.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=34442], [object_number=209, tag_tid=14, entry_tid=34441], [object_number=208, tag_tid=14, entry_tid=34440], [object_number=211, tag_tid=14, entry_tid=34439]]], state=connected] 2026-02-19T20:36:09.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=210, tag_tid=14, entry_tid=34442' 2026-02-19T20:36:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=210, tag_tid=14, entry_tid=34442' 2026-02-19T20:36:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=34442' '!=' 'object_number=209, tag_tid=14, entry_tid=34161' 2026-02-19T20:36:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=34442' = 'object_number=209, tag_tid=14, entry_tid=34161' 2026-02-19T20:36:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=210, tag_tid=14, entry_tid=34442' 2026-02-19T20:36:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=34442' 2026-02-19T20:36:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=34442' 2026-02-19T20:36:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34442 2026-02-19T20:36:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 34442 2026-02-19T20:36:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:09.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:09.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:09.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:09.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:09.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:09.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:09.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:09.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:09.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:09.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:09.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:09.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:09.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:09.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:09.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:09.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:10.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=34732], [object_number=211, tag_tid=14, entry_tid=34731], [object_number=210, tag_tid=14, entry_tid=34730], [object_number=209, tag_tid=14, entry_tid=34729]]], state=connected] 2026-02-19T20:36:10.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 52 2026-02-19T20:36:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:10.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=34732], [object_number=211, tag_tid=14, entry_tid=34731], [object_number=210, tag_tid=14, entry_tid=34730], [object_number=209, tag_tid=14, entry_tid=34729]]], state=connected] 2026-02-19T20:36:10.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=34732' 2026-02-19T20:36:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=34732' 2026-02-19T20:36:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=34732' '!=' 'object_number=210, tag_tid=14, entry_tid=34442' 2026-02-19T20:36:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=34732' = 'object_number=210, tag_tid=14, entry_tid=34442' 2026-02-19T20:36:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=34732' 2026-02-19T20:36:10.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:10.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:10.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:10.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=34732' 2026-02-19T20:36:10.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:10.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:10.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=34732' 2026-02-19T20:36:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:10.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34732 2026-02-19T20:36:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 34732 2026-02-19T20:36:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:10.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:10.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:10.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:10.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:10.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:10.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:10.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:10.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:10.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:10.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:10.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:10.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:10.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:10.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:10.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=34913], [object_number=212, tag_tid=14, entry_tid=34912], [object_number=215, tag_tid=14, entry_tid=34911], [object_number=214, tag_tid=14, entry_tid=34910]]], state=connected] 2026-02-19T20:36:10.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:10.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:10.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:10.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:10.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:10.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:10.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:10.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:10.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=34913], [object_number=212, tag_tid=14, entry_tid=34912], [object_number=215, tag_tid=14, entry_tid=34911], [object_number=214, tag_tid=14, entry_tid=34910]]], state=connected] 2026-02-19T20:36:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=213, tag_tid=14, entry_tid=34913' 2026-02-19T20:36:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=213, tag_tid=14, entry_tid=34913' 2026-02-19T20:36:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=34913' '!=' 'object_number=208, tag_tid=14, entry_tid=34732' 2026-02-19T20:36:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=34913' = 'object_number=208, tag_tid=14, entry_tid=34732' 2026-02-19T20:36:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=213, tag_tid=14, entry_tid=34913' 2026-02-19T20:36:10.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:10.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:10.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=34913' 2026-02-19T20:36:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:10.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:10.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:10.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:10.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:10.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:10.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=34913' 2026-02-19T20:36:10.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:10.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:10.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34913 2026-02-19T20:36:10.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 34913 2026-02-19T20:36:10.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:10.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:10.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:10.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:10.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:10.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=35208], [object_number=215, tag_tid=14, entry_tid=35207], [object_number=214, tag_tid=14, entry_tid=35206], [object_number=213, tag_tid=14, entry_tid=35205]]], state=connected] 2026-02-19T20:36:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:10.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:10.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:10.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:10.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:10.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:10.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=35208], [object_number=215, tag_tid=14, entry_tid=35207], [object_number=214, tag_tid=14, entry_tid=35206], [object_number=213, tag_tid=14, entry_tid=35205]]], state=connected] 2026-02-19T20:36:11.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=212, tag_tid=14, entry_tid=35208' 2026-02-19T20:36:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=212, tag_tid=14, entry_tid=35208' 2026-02-19T20:36:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=35208' '!=' 'object_number=213, tag_tid=14, entry_tid=34913' 2026-02-19T20:36:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=35208' = 'object_number=213, tag_tid=14, entry_tid=34913' 2026-02-19T20:36:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=212, tag_tid=14, entry_tid=35208' 2026-02-19T20:36:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=35208' 2026-02-19T20:36:11.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:11.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:11.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:11.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:11.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:11.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:11.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:11.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=35208' 2026-02-19T20:36:11.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:11.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:11.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35208 2026-02-19T20:36:11.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 35208 2026-02-19T20:36:11.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:11.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:11.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:11.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:11.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:11.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:11.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:11.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:11.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:11.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:11.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:11.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:11.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:11.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=35653], [object_number=212, tag_tid=14, entry_tid=35652], [object_number=215, tag_tid=14, entry_tid=35651], [object_number=214, tag_tid=14, entry_tid=35650]]], state=connected] 2026-02-19T20:36:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=35653], [object_number=212, tag_tid=14, entry_tid=35652], [object_number=215, tag_tid=14, entry_tid=35651], [object_number=214, tag_tid=14, entry_tid=35650]]], state=connected] 2026-02-19T20:36:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=213, tag_tid=14, entry_tid=35653' 2026-02-19T20:36:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=213, tag_tid=14, entry_tid=35653' 2026-02-19T20:36:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=35653' '!=' 'object_number=212, tag_tid=14, entry_tid=35208' 2026-02-19T20:36:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=35653' = 'object_number=212, tag_tid=14, entry_tid=35208' 2026-02-19T20:36:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=213, tag_tid=14, entry_tid=35653' 2026-02-19T20:36:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:11.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:11.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=35653' 2026-02-19T20:36:11.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:11.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:11.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:11.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:11.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:11.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=35653' 2026-02-19T20:36:11.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:11.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35653 2026-02-19T20:36:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 35653 2026-02-19T20:36:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:11.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:11.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:11.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:11.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:11.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:11.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:11.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:11.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:11.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:11.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:11.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:11.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:11.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:11.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:11.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:11.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=35967], [object_number=214, tag_tid=14, entry_tid=35966], [object_number=213, tag_tid=14, entry_tid=35965], [object_number=212, tag_tid=14, entry_tid=35964]]], state=connected] 2026-02-19T20:36:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:12.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:12.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=35967], [object_number=214, tag_tid=14, entry_tid=35966], [object_number=213, tag_tid=14, entry_tid=35965], [object_number=212, tag_tid=14, entry_tid=35964]]], state=connected] 2026-02-19T20:36:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=35967' 2026-02-19T20:36:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=35967' 2026-02-19T20:36:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=35967' '!=' 'object_number=213, tag_tid=14, entry_tid=35653' 2026-02-19T20:36:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=35967' = 'object_number=213, tag_tid=14, entry_tid=35653' 2026-02-19T20:36:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=35967' 2026-02-19T20:36:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:12.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=35967' 2026-02-19T20:36:12.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:12.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=35967' 2026-02-19T20:36:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35967 2026-02-19T20:36:12.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 35967 2026-02-19T20:36:12.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:12.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:12.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:12.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:12.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:12.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:12.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:12.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:12.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:12.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:12.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:12.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:12.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=36380], [object_number=215, tag_tid=14, entry_tid=36379], [object_number=214, tag_tid=14, entry_tid=36378], [object_number=213, tag_tid=14, entry_tid=36377]]], state=connected] 2026-02-19T20:36:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:12.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=36380], [object_number=215, tag_tid=14, entry_tid=36379], [object_number=214, tag_tid=14, entry_tid=36378], [object_number=213, tag_tid=14, entry_tid=36377]]], state=connected] 2026-02-19T20:36:12.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=212, tag_tid=14, entry_tid=36380' 2026-02-19T20:36:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=212, tag_tid=14, entry_tid=36380' 2026-02-19T20:36:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=36380' '!=' 'object_number=215, tag_tid=14, entry_tid=35967' 2026-02-19T20:36:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=36380' = 'object_number=215, tag_tid=14, entry_tid=35967' 2026-02-19T20:36:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=212, tag_tid=14, entry_tid=36380' 2026-02-19T20:36:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:12.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=36380' 2026-02-19T20:36:12.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:12.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=36380' 2026-02-19T20:36:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36380 2026-02-19T20:36:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 36380 2026-02-19T20:36:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:12.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:12.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:12.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:12.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:12.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:12.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:12.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:12.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:12.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:12.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:12.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:12.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:12.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:12.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:12.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:12.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:12.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:12.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:12.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=36660], [object_number=215, tag_tid=14, entry_tid=36659], [object_number=214, tag_tid=14, entry_tid=36658], [object_number=213, tag_tid=14, entry_tid=36657]]], state=connected] 2026-02-19T20:36:12.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:12.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:12.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:12.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:12.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:12.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:12.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:12.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:12.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:12.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:12.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:12.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:12.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=36660], [object_number=215, tag_tid=14, entry_tid=36659], [object_number=214, tag_tid=14, entry_tid=36658], [object_number=213, tag_tid=14, entry_tid=36657]]], state=connected] 2026-02-19T20:36:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=212, tag_tid=14, entry_tid=36660' 2026-02-19T20:36:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=212, tag_tid=14, entry_tid=36660' 2026-02-19T20:36:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=36660' '!=' 'object_number=212, tag_tid=14, entry_tid=36380' 2026-02-19T20:36:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=36660' = 'object_number=212, tag_tid=14, entry_tid=36380' 2026-02-19T20:36:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=212, tag_tid=14, entry_tid=36660' 2026-02-19T20:36:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:12.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=36660' 2026-02-19T20:36:12.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:12.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=36660' 2026-02-19T20:36:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:12.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:12.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36660 2026-02-19T20:36:12.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 36660 2026-02-19T20:36:12.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:12.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:12.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:13.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:13.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:13.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:13.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=36967], [object_number=214, tag_tid=14, entry_tid=36966], [object_number=213, tag_tid=14, entry_tid=36965], [object_number=212, tag_tid=14, entry_tid=36964]]], state=connected] 2026-02-19T20:36:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:13.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:13.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=36967], [object_number=214, tag_tid=14, entry_tid=36966], [object_number=213, tag_tid=14, entry_tid=36965], [object_number=212, tag_tid=14, entry_tid=36964]]], state=connected] 2026-02-19T20:36:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=36967' 2026-02-19T20:36:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=36967' 2026-02-19T20:36:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=36967' '!=' 'object_number=212, tag_tid=14, entry_tid=36660' 2026-02-19T20:36:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=36967' = 'object_number=212, tag_tid=14, entry_tid=36660' 2026-02-19T20:36:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=36967' 2026-02-19T20:36:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=36967' 2026-02-19T20:36:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=36967' 2026-02-19T20:36:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36967 2026-02-19T20:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 36967 2026-02-19T20:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:13.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:13.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:13.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:13.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:13.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:13.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:13.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:13.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:13.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:13.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:13.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:13.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:13.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=37269], [object_number=212, tag_tid=14, entry_tid=37268], [object_number=215, tag_tid=14, entry_tid=37267], [object_number=214, tag_tid=14, entry_tid=37266]]], state=connected] 2026-02-19T20:36:13.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=37269], [object_number=212, tag_tid=14, entry_tid=37268], [object_number=215, tag_tid=14, entry_tid=37267], [object_number=214, tag_tid=14, entry_tid=37266]]], state=connected] 2026-02-19T20:36:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=213, tag_tid=14, entry_tid=37269' 2026-02-19T20:36:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=213, tag_tid=14, entry_tid=37269' 2026-02-19T20:36:13.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=37269' '!=' 'object_number=215, tag_tid=14, entry_tid=36967' 2026-02-19T20:36:13.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:13.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=37269' = 'object_number=215, tag_tid=14, entry_tid=36967' 2026-02-19T20:36:13.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=213, tag_tid=14, entry_tid=37269' 2026-02-19T20:36:13.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=37269' 2026-02-19T20:36:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=37269' 2026-02-19T20:36:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37269 2026-02-19T20:36:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 37269 2026-02-19T20:36:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:13.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:13.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:13.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:13.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:14.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:14.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:14.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:14.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:14.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:14.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:14.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:14.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:14.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:14.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:14.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=37543], [object_number=214, tag_tid=14, entry_tid=37542], [object_number=213, tag_tid=14, entry_tid=37541], [object_number=212, tag_tid=14, entry_tid=37540]]], state=connected] 2026-02-19T20:36:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:14.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:14.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=37543], [object_number=214, tag_tid=14, entry_tid=37542], [object_number=213, tag_tid=14, entry_tid=37541], [object_number=212, tag_tid=14, entry_tid=37540]]], state=connected] 2026-02-19T20:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=37543' 2026-02-19T20:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=37543' 2026-02-19T20:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=37543' '!=' 'object_number=213, tag_tid=14, entry_tid=37269' 2026-02-19T20:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=37543' = 'object_number=213, tag_tid=14, entry_tid=37269' 2026-02-19T20:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=37543' 2026-02-19T20:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=37543' 2026-02-19T20:36:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:14.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:14.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=37543' 2026-02-19T20:36:14.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:14.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37543 2026-02-19T20:36:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 37543 2026-02-19T20:36:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:14.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:14.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:14.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:14.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:14.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:14.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:14.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:14.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:14.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:14.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:14.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=37807], [object_number=214, tag_tid=14, entry_tid=37806], [object_number=213, tag_tid=14, entry_tid=37805], [object_number=212, tag_tid=14, entry_tid=37804]]], state=connected] 2026-02-19T20:36:14.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=37807], [object_number=214, tag_tid=14, entry_tid=37806], [object_number=213, tag_tid=14, entry_tid=37805], [object_number=212, tag_tid=14, entry_tid=37804]]], state=connected] 2026-02-19T20:36:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=37807' 2026-02-19T20:36:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=37807' 2026-02-19T20:36:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=37807' '!=' 'object_number=215, tag_tid=14, entry_tid=37543' 2026-02-19T20:36:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=37807' = 'object_number=215, tag_tid=14, entry_tid=37543' 2026-02-19T20:36:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=37807' 2026-02-19T20:36:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=37807' 2026-02-19T20:36:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:14.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:14.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:14.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:14.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:14.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=37807' 2026-02-19T20:36:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:14.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37807 2026-02-19T20:36:14.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 37807 2026-02-19T20:36:14.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:14.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:14.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:14.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:14.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:14.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:14.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:14.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:14.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:14.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=38121], [object_number=212, tag_tid=14, entry_tid=38120], [object_number=215, tag_tid=14, entry_tid=38119], [object_number=214, tag_tid=14, entry_tid=38118]]], state=connected] 2026-02-19T20:36:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=38121], [object_number=212, tag_tid=14, entry_tid=38120], [object_number=215, tag_tid=14, entry_tid=38119], [object_number=214, tag_tid=14, entry_tid=38118]]], state=connected] 2026-02-19T20:36:15.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=213, tag_tid=14, entry_tid=38121' 2026-02-19T20:36:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=213, tag_tid=14, entry_tid=38121' 2026-02-19T20:36:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=38121' '!=' 'object_number=215, tag_tid=14, entry_tid=37807' 2026-02-19T20:36:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=38121' = 'object_number=215, tag_tid=14, entry_tid=37807' 2026-02-19T20:36:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=213, tag_tid=14, entry_tid=38121' 2026-02-19T20:36:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:15.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=38121' 2026-02-19T20:36:15.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:15.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=38121' 2026-02-19T20:36:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:15.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38121 2026-02-19T20:36:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 38121 2026-02-19T20:36:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:15.100 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.337676469316301) > weight (0.25) 2026-02-19T20:36:15.101 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.46682141410502676) > weight (0.25) 2026-02-19T20:36:15.101 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9078487642734471) > weight (0.25) 2026-02-19T20:36:15.101 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.9181181740479274) > weight (0.25) 2026-02-19T20:36:15.101 INFO:tasks.rbd_mirror_thrash:waiting for 76.61000923001208 secs before thrashing 2026-02-19T20:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:15.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:15.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:15.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:15.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:15.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=38423], [object_number=214, tag_tid=14, entry_tid=38422], [object_number=213, tag_tid=14, entry_tid=38421], [object_number=212, tag_tid=14, entry_tid=38420]]], state=connected] 2026-02-19T20:36:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:15.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:15.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:15.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:15.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=38423], [object_number=214, tag_tid=14, entry_tid=38422], [object_number=213, tag_tid=14, entry_tid=38421], [object_number=212, tag_tid=14, entry_tid=38420]]], state=connected] 2026-02-19T20:36:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=38423' 2026-02-19T20:36:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=38423' 2026-02-19T20:36:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=38423' '!=' 'object_number=213, tag_tid=14, entry_tid=38121' 2026-02-19T20:36:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=38423' = 'object_number=213, tag_tid=14, entry_tid=38121' 2026-02-19T20:36:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=38423' 2026-02-19T20:36:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=38423' 2026-02-19T20:36:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=38423' 2026-02-19T20:36:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38423 2026-02-19T20:36:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 38423 2026-02-19T20:36:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:15.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:15.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:15.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:15.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:15.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:15.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:15.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:15.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:15.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:15.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=38727], [object_number=214, tag_tid=14, entry_tid=38726], [object_number=213, tag_tid=14, entry_tid=38725], [object_number=212, tag_tid=14, entry_tid=38724]]], state=connected] 2026-02-19T20:36:15.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:15.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:15.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:15.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:15.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:15.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:15.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:15.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=38727], [object_number=214, tag_tid=14, entry_tid=38726], [object_number=213, tag_tid=14, entry_tid=38725], [object_number=212, tag_tid=14, entry_tid=38724]]], state=connected] 2026-02-19T20:36:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=38727' 2026-02-19T20:36:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=38727' 2026-02-19T20:36:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=38727' '!=' 'object_number=215, tag_tid=14, entry_tid=38423' 2026-02-19T20:36:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=38727' = 'object_number=215, tag_tid=14, entry_tid=38423' 2026-02-19T20:36:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=38727' 2026-02-19T20:36:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=38727' 2026-02-19T20:36:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:15.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=38727' 2026-02-19T20:36:15.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:15.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38727 2026-02-19T20:36:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 38727 2026-02-19T20:36:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=39006], [object_number=213, tag_tid=14, entry_tid=39005], [object_number=212, tag_tid=14, entry_tid=39004], [object_number=215, tag_tid=14, entry_tid=39003]]], state=connected] 2026-02-19T20:36:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 53 2026-02-19T20:36:16.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:16.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:16.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:16.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=39006], [object_number=213, tag_tid=14, entry_tid=39005], [object_number=212, tag_tid=14, entry_tid=39004], [object_number=215, tag_tid=14, entry_tid=39003]]], state=connected] 2026-02-19T20:36:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=214, tag_tid=14, entry_tid=39006' 2026-02-19T20:36:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=214, tag_tid=14, entry_tid=39006' 2026-02-19T20:36:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=39006' '!=' 'object_number=215, tag_tid=14, entry_tid=38727' 2026-02-19T20:36:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=39006' = 'object_number=215, tag_tid=14, entry_tid=38727' 2026-02-19T20:36:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=214, tag_tid=14, entry_tid=39006' 2026-02-19T20:36:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=39006' 2026-02-19T20:36:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:16.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=39006' 2026-02-19T20:36:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39006 2026-02-19T20:36:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 39006 2026-02-19T20:36:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:16.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:16.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:16.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:16.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:16.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:16.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:16.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=39222], [object_number=217, tag_tid=14, entry_tid=39221], [object_number=216, tag_tid=14, entry_tid=39220], [object_number=219, tag_tid=14, entry_tid=39219]]], state=connected] 2026-02-19T20:36:16.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:16.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:16.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:16.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:16.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=39222], [object_number=217, tag_tid=14, entry_tid=39221], [object_number=216, tag_tid=14, entry_tid=39220], [object_number=219, tag_tid=14, entry_tid=39219]]], state=connected] 2026-02-19T20:36:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:16.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=218, tag_tid=14, entry_tid=39222' 2026-02-19T20:36:16.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=218, tag_tid=14, entry_tid=39222' 2026-02-19T20:36:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=39222' '!=' 'object_number=214, tag_tid=14, entry_tid=39006' 2026-02-19T20:36:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=39222' = 'object_number=214, tag_tid=14, entry_tid=39006' 2026-02-19T20:36:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=218, tag_tid=14, entry_tid=39222' 2026-02-19T20:36:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=39222' 2026-02-19T20:36:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:16.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:16.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:16.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:16.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=39222' 2026-02-19T20:36:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39222 2026-02-19T20:36:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 39222 2026-02-19T20:36:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:17.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:17.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=39647], [object_number=218, tag_tid=14, entry_tid=39646], [object_number=217, tag_tid=14, entry_tid=39645], [object_number=216, tag_tid=14, entry_tid=39644]]], state=connected] 2026-02-19T20:36:17.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:17.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=39647], [object_number=218, tag_tid=14, entry_tid=39646], [object_number=217, tag_tid=14, entry_tid=39645], [object_number=216, tag_tid=14, entry_tid=39644]]], state=connected] 2026-02-19T20:36:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=219, tag_tid=14, entry_tid=39647' 2026-02-19T20:36:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=219, tag_tid=14, entry_tid=39647' 2026-02-19T20:36:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=39647' '!=' 'object_number=218, tag_tid=14, entry_tid=39222' 2026-02-19T20:36:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=39647' = 'object_number=218, tag_tid=14, entry_tid=39222' 2026-02-19T20:36:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=219, tag_tid=14, entry_tid=39647' 2026-02-19T20:36:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=39647' 2026-02-19T20:36:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=39647' 2026-02-19T20:36:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39647 2026-02-19T20:36:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 39647 2026-02-19T20:36:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:17.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:17.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:17.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:17.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:17.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:17.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:17.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:17.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:17.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:17.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:17.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:17.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:17.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=39931], [object_number=218, tag_tid=14, entry_tid=39930], [object_number=217, tag_tid=14, entry_tid=39929], [object_number=216, tag_tid=14, entry_tid=39928]]], state=connected] 2026-02-19T20:36:17.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:17.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:17.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=39931], [object_number=218, tag_tid=14, entry_tid=39930], [object_number=217, tag_tid=14, entry_tid=39929], [object_number=216, tag_tid=14, entry_tid=39928]]], state=connected] 2026-02-19T20:36:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=219, tag_tid=14, entry_tid=39931' 2026-02-19T20:36:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=219, tag_tid=14, entry_tid=39931' 2026-02-19T20:36:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=39931' '!=' 'object_number=219, tag_tid=14, entry_tid=39647' 2026-02-19T20:36:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=39931' = 'object_number=219, tag_tid=14, entry_tid=39647' 2026-02-19T20:36:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=219, tag_tid=14, entry_tid=39931' 2026-02-19T20:36:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=39931' 2026-02-19T20:36:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:17.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:17.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:17.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:17.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:17.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:17.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=39931' 2026-02-19T20:36:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39931 2026-02-19T20:36:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 39931 2026-02-19T20:36:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:18.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:18.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:18.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:18.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:18.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:18.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:18.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:18.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:18.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:18.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:18.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:18.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:18.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:18.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=40224], [object_number=219, tag_tid=14, entry_tid=40223], [object_number=218, tag_tid=14, entry_tid=40222], [object_number=217, tag_tid=14, entry_tid=40221]]], state=connected] 2026-02-19T20:36:18.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:18.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:18.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:18.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:18.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:18.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:18.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=40224], [object_number=219, tag_tid=14, entry_tid=40223], [object_number=218, tag_tid=14, entry_tid=40222], [object_number=217, tag_tid=14, entry_tid=40221]]], state=connected] 2026-02-19T20:36:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=40224' 2026-02-19T20:36:18.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=40224' 2026-02-19T20:36:18.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=40224' '!=' 'object_number=219, tag_tid=14, entry_tid=39931' 2026-02-19T20:36:18.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:18.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=40224' = 'object_number=219, tag_tid=14, entry_tid=39931' 2026-02-19T20:36:18.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=40224' 2026-02-19T20:36:18.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:18.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:18.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:18.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=40224' 2026-02-19T20:36:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=40224' 2026-02-19T20:36:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40224 2026-02-19T20:36:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 40224 2026-02-19T20:36:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:18.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:18.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:18.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:18.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:18.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:18.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:18.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:18.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:18.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:18.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:18.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:18.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:18.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:18.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:18.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:18.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=40478], [object_number=217, tag_tid=14, entry_tid=40477], [object_number=216, tag_tid=14, entry_tid=40476], [object_number=219, tag_tid=14, entry_tid=40475]]], state=connected] 2026-02-19T20:36:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:18.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:18.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:18.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:18.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=40478], [object_number=217, tag_tid=14, entry_tid=40477], [object_number=216, tag_tid=14, entry_tid=40476], [object_number=219, tag_tid=14, entry_tid=40475]]], state=connected] 2026-02-19T20:36:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:18.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=218, tag_tid=14, entry_tid=40478' 2026-02-19T20:36:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=218, tag_tid=14, entry_tid=40478' 2026-02-19T20:36:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=40478' '!=' 'object_number=216, tag_tid=14, entry_tid=40224' 2026-02-19T20:36:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=40478' = 'object_number=216, tag_tid=14, entry_tid=40224' 2026-02-19T20:36:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=218, tag_tid=14, entry_tid=40478' 2026-02-19T20:36:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:18.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=40478' 2026-02-19T20:36:18.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:18.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:18.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=40478' 2026-02-19T20:36:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:18.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40478 2026-02-19T20:36:18.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 40478 2026-02-19T20:36:18.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:18.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:18.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:18.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:18.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:18.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:19.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:19.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:19.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:19.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:19.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:19.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=40755], [object_number=218, tag_tid=14, entry_tid=40754], [object_number=217, tag_tid=14, entry_tid=40753], [object_number=216, tag_tid=14, entry_tid=40752]]], state=connected] 2026-02-19T20:36:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=40755], [object_number=218, tag_tid=14, entry_tid=40754], [object_number=217, tag_tid=14, entry_tid=40753], [object_number=216, tag_tid=14, entry_tid=40752]]], state=connected] 2026-02-19T20:36:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=219, tag_tid=14, entry_tid=40755' 2026-02-19T20:36:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=219, tag_tid=14, entry_tid=40755' 2026-02-19T20:36:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=40755' '!=' 'object_number=218, tag_tid=14, entry_tid=40478' 2026-02-19T20:36:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=40755' = 'object_number=218, tag_tid=14, entry_tid=40478' 2026-02-19T20:36:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=219, tag_tid=14, entry_tid=40755' 2026-02-19T20:36:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=40755' 2026-02-19T20:36:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=40755' 2026-02-19T20:36:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40755 2026-02-19T20:36:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 40755 2026-02-19T20:36:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:19.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:19.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:19.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:19.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:19.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:19.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:19.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:19.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:19.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:19.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:19.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:19.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:19.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:19.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:19.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:19.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:19.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:19.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:19.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:19.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=41064], [object_number=219, tag_tid=14, entry_tid=41063], [object_number=218, tag_tid=14, entry_tid=41062], [object_number=217, tag_tid=14, entry_tid=41061]]], state=connected] 2026-02-19T20:36:19.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:19.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:19.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:19.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:19.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:19.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=41064], [object_number=219, tag_tid=14, entry_tid=41063], [object_number=218, tag_tid=14, entry_tid=41062], [object_number=217, tag_tid=14, entry_tid=41061]]], state=connected] 2026-02-19T20:36:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=41064' 2026-02-19T20:36:19.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=41064' 2026-02-19T20:36:19.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=41064' '!=' 'object_number=219, tag_tid=14, entry_tid=40755' 2026-02-19T20:36:19.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:19.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=41064' = 'object_number=219, tag_tid=14, entry_tid=40755' 2026-02-19T20:36:19.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=41064' 2026-02-19T20:36:19.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:19.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:19.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:19.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=41064' 2026-02-19T20:36:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:19.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:19.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:19.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=41064' 2026-02-19T20:36:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41064 2026-02-19T20:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 41064 2026-02-19T20:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:19.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:19.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:19.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:19.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:19.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:19.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:19.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:19.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:19.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:19.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:19.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=41358], [object_number=217, tag_tid=14, entry_tid=41357], [object_number=216, tag_tid=14, entry_tid=41356], [object_number=219, tag_tid=14, entry_tid=41355]]], state=connected] 2026-02-19T20:36:19.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:19.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=41358], [object_number=217, tag_tid=14, entry_tid=41357], [object_number=216, tag_tid=14, entry_tid=41356], [object_number=219, tag_tid=14, entry_tid=41355]]], state=connected] 2026-02-19T20:36:19.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=218, tag_tid=14, entry_tid=41358' 2026-02-19T20:36:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=218, tag_tid=14, entry_tid=41358' 2026-02-19T20:36:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=41358' '!=' 'object_number=216, tag_tid=14, entry_tid=41064' 2026-02-19T20:36:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=41358' = 'object_number=216, tag_tid=14, entry_tid=41064' 2026-02-19T20:36:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=218, tag_tid=14, entry_tid=41358' 2026-02-19T20:36:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=41358' 2026-02-19T20:36:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=41358' 2026-02-19T20:36:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41358 2026-02-19T20:36:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 41358 2026-02-19T20:36:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:20.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:20.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:20.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:20.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:20.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:20.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:20.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:20.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:20.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:20.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:20.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:20.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:20.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:20.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:20.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:20.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:20.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:20.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:20.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=41637], [object_number=216, tag_tid=14, entry_tid=41636], [object_number=219, tag_tid=14, entry_tid=41635], [object_number=218, tag_tid=14, entry_tid=41634]]], state=connected] 2026-02-19T20:36:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=41637], [object_number=216, tag_tid=14, entry_tid=41636], [object_number=219, tag_tid=14, entry_tid=41635], [object_number=218, tag_tid=14, entry_tid=41634]]], state=connected] 2026-02-19T20:36:20.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=41637' 2026-02-19T20:36:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=41637' 2026-02-19T20:36:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=41637' '!=' 'object_number=218, tag_tid=14, entry_tid=41358' 2026-02-19T20:36:20.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:20.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=41637' = 'object_number=218, tag_tid=14, entry_tid=41358' 2026-02-19T20:36:20.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=217, tag_tid=14, entry_tid=41637' 2026-02-19T20:36:20.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:20.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=41637' 2026-02-19T20:36:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=41637' 2026-02-19T20:36:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:20.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41637 2026-02-19T20:36:20.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 41637 2026-02-19T20:36:20.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:20.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:20.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:20.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:20.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:20.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:20.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:20.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:20.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:20.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:20.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:20.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:20.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:20.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:20.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:20.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:20.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:20.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:20.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:20.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:20.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:20.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:20.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:20.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:20.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:20.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:20.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:20.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:20.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:20.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=41950], [object_number=217, tag_tid=14, entry_tid=41949], [object_number=216, tag_tid=14, entry_tid=41948], [object_number=219, tag_tid=14, entry_tid=41947]]], state=connected] 2026-02-19T20:36:20.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:20.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:20.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:20.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:20.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=41950], [object_number=217, tag_tid=14, entry_tid=41949], [object_number=216, tag_tid=14, entry_tid=41948], [object_number=219, tag_tid=14, entry_tid=41947]]], state=connected] 2026-02-19T20:36:20.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=218, tag_tid=14, entry_tid=41950' 2026-02-19T20:36:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=218, tag_tid=14, entry_tid=41950' 2026-02-19T20:36:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=41950' '!=' 'object_number=217, tag_tid=14, entry_tid=41637' 2026-02-19T20:36:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=41950' = 'object_number=217, tag_tid=14, entry_tid=41637' 2026-02-19T20:36:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=218, tag_tid=14, entry_tid=41950' 2026-02-19T20:36:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:20.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:20.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=41950' 2026-02-19T20:36:20.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:20.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:20.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:20.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:20.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:20.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:20.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=41950' 2026-02-19T20:36:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41950 2026-02-19T20:36:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 41950 2026-02-19T20:36:20.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:20.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:20.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:20.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:20.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:21.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:21.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=42246], [object_number=217, tag_tid=14, entry_tid=42245], [object_number=216, tag_tid=14, entry_tid=42244], [object_number=219, tag_tid=14, entry_tid=42243]]], state=connected] 2026-02-19T20:36:21.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:21.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=42246], [object_number=217, tag_tid=14, entry_tid=42245], [object_number=216, tag_tid=14, entry_tid=42244], [object_number=219, tag_tid=14, entry_tid=42243]]], state=connected] 2026-02-19T20:36:21.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=218, tag_tid=14, entry_tid=42246' 2026-02-19T20:36:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=218, tag_tid=14, entry_tid=42246' 2026-02-19T20:36:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=42246' '!=' 'object_number=218, tag_tid=14, entry_tid=41950' 2026-02-19T20:36:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=42246' = 'object_number=218, tag_tid=14, entry_tid=41950' 2026-02-19T20:36:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=218, tag_tid=14, entry_tid=42246' 2026-02-19T20:36:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:21.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:21.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:21.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=42246' 2026-02-19T20:36:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:21.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:21.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:21.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:21.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:21.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:21.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=42246' 2026-02-19T20:36:21.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:21.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42246 2026-02-19T20:36:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 42246 2026-02-19T20:36:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:21.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:21.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:21.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:21.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:21.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:21.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:21.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:21.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:21.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:21.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:21.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:21.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:21.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:21.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:21.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:21.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:21.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:21.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:21.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:21.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:21.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=42659], [object_number=218, tag_tid=14, entry_tid=42658], [object_number=217, tag_tid=14, entry_tid=42657], [object_number=216, tag_tid=14, entry_tid=42656]]], state=connected] 2026-02-19T20:36:21.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:21.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:21.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:21.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:21.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:21.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=42659], [object_number=218, tag_tid=14, entry_tid=42658], [object_number=217, tag_tid=14, entry_tid=42657], [object_number=216, tag_tid=14, entry_tid=42656]]], state=connected] 2026-02-19T20:36:22.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=219, tag_tid=14, entry_tid=42659' 2026-02-19T20:36:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=219, tag_tid=14, entry_tid=42659' 2026-02-19T20:36:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=42659' '!=' 'object_number=218, tag_tid=14, entry_tid=42246' 2026-02-19T20:36:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=42659' = 'object_number=218, tag_tid=14, entry_tid=42246' 2026-02-19T20:36:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=219, tag_tid=14, entry_tid=42659' 2026-02-19T20:36:22.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:22.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:22.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:22.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=42659' 2026-02-19T20:36:22.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:22.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=42659' 2026-02-19T20:36:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42659 2026-02-19T20:36:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 42659 2026-02-19T20:36:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:22.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:22.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:22.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:22.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:22.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:22.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:22.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=43100], [object_number=219, tag_tid=14, entry_tid=43099], [object_number=218, tag_tid=14, entry_tid=43098], [object_number=217, tag_tid=14, entry_tid=43097]]], state=connected] 2026-02-19T20:36:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 54 2026-02-19T20:36:22.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:22.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:22.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=43100], [object_number=219, tag_tid=14, entry_tid=43099], [object_number=218, tag_tid=14, entry_tid=43098], [object_number=217, tag_tid=14, entry_tid=43097]]], state=connected] 2026-02-19T20:36:22.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=43100' 2026-02-19T20:36:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=43100' 2026-02-19T20:36:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=43100' '!=' 'object_number=219, tag_tid=14, entry_tid=42659' 2026-02-19T20:36:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=43100' = 'object_number=219, tag_tid=14, entry_tid=42659' 2026-02-19T20:36:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=43100' 2026-02-19T20:36:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=43100' 2026-02-19T20:36:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=43100' 2026-02-19T20:36:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43100 2026-02-19T20:36:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 43100 2026-02-19T20:36:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:22.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:22.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:22.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:22.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:22.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=43354], [object_number=221, tag_tid=14, entry_tid=43353], [object_number=220, tag_tid=14, entry_tid=43352], [object_number=223, tag_tid=14, entry_tid=43351]]], state=connected] 2026-02-19T20:36:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:22.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:22.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:22.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:22.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:22.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=43354], [object_number=221, tag_tid=14, entry_tid=43353], [object_number=220, tag_tid=14, entry_tid=43352], [object_number=223, tag_tid=14, entry_tid=43351]]], state=connected] 2026-02-19T20:36:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=222, tag_tid=14, entry_tid=43354' 2026-02-19T20:36:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=222, tag_tid=14, entry_tid=43354' 2026-02-19T20:36:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=43354' '!=' 'object_number=216, tag_tid=14, entry_tid=43100' 2026-02-19T20:36:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=43354' = 'object_number=216, tag_tid=14, entry_tid=43100' 2026-02-19T20:36:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=222, tag_tid=14, entry_tid=43354' 2026-02-19T20:36:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=43354' 2026-02-19T20:36:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=43354' 2026-02-19T20:36:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43354 2026-02-19T20:36:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 43354 2026-02-19T20:36:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:23.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:23.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:23.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:23.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:23.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:23.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:23.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:23.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:23.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:23.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:23.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:23.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=43663], [object_number=222, tag_tid=14, entry_tid=43662], [object_number=221, tag_tid=14, entry_tid=43661], [object_number=220, tag_tid=14, entry_tid=43660]]], state=connected] 2026-02-19T20:36:23.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=43663], [object_number=222, tag_tid=14, entry_tid=43662], [object_number=221, tag_tid=14, entry_tid=43661], [object_number=220, tag_tid=14, entry_tid=43660]]], state=connected] 2026-02-19T20:36:23.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=223, tag_tid=14, entry_tid=43663' 2026-02-19T20:36:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=223, tag_tid=14, entry_tid=43663' 2026-02-19T20:36:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=43663' '!=' 'object_number=222, tag_tid=14, entry_tid=43354' 2026-02-19T20:36:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=43663' = 'object_number=222, tag_tid=14, entry_tid=43354' 2026-02-19T20:36:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=223, tag_tid=14, entry_tid=43663' 2026-02-19T20:36:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=43663' 2026-02-19T20:36:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:23.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=43663' 2026-02-19T20:36:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:23.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43663 2026-02-19T20:36:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 43663 2026-02-19T20:36:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:23.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:23.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:23.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:23.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:23.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:23.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:23.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:23.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:23.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:23.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:23.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:23.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:23.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:23.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:23.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:23.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=44073], [object_number=220, tag_tid=14, entry_tid=44072], [object_number=223, tag_tid=14, entry_tid=44071], [object_number=222, tag_tid=14, entry_tid=44070]]], state=connected] 2026-02-19T20:36:23.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:23.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:23.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:23.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:23.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:23.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:23.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:23.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:23.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:23.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:23.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:23.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:23.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=44073], [object_number=220, tag_tid=14, entry_tid=44072], [object_number=223, tag_tid=14, entry_tid=44071], [object_number=222, tag_tid=14, entry_tid=44070]]], state=connected] 2026-02-19T20:36:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=221, tag_tid=14, entry_tid=44073' 2026-02-19T20:36:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=221, tag_tid=14, entry_tid=44073' 2026-02-19T20:36:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=44073' '!=' 'object_number=223, tag_tid=14, entry_tid=43663' 2026-02-19T20:36:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=44073' = 'object_number=223, tag_tid=14, entry_tid=43663' 2026-02-19T20:36:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=221, tag_tid=14, entry_tid=44073' 2026-02-19T20:36:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=44073' 2026-02-19T20:36:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=44073' 2026-02-19T20:36:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44073 2026-02-19T20:36:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 44073 2026-02-19T20:36:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:24.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:24.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:24.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:24.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:24.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:24.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:24.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:24.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:24.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:24.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:24.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:24.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:24.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:24.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:24.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:24.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:24.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=44346], [object_number=221, tag_tid=14, entry_tid=44345], [object_number=220, tag_tid=14, entry_tid=44344], [object_number=223, tag_tid=14, entry_tid=44343]]], state=connected] 2026-02-19T20:36:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:24.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:24.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:24.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:24.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:24.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:24.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=222, tag_tid=14, entry_tid=44346], [object_number=221, tag_tid=14, entry_tid=44345], [object_number=220, tag_tid=14, entry_tid=44344], [object_number=223, tag_tid=14, entry_tid=44343]]], state=connected] 2026-02-19T20:36:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=222, tag_tid=14, entry_tid=44346' 2026-02-19T20:36:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=222, tag_tid=14, entry_tid=44346' 2026-02-19T20:36:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=44346' '!=' 'object_number=221, tag_tid=14, entry_tid=44073' 2026-02-19T20:36:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=222, tag_tid=14, entry_tid=44346' = 'object_number=221, tag_tid=14, entry_tid=44073' 2026-02-19T20:36:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=222, tag_tid=14, entry_tid=44346' 2026-02-19T20:36:24.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:24.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:24.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:24.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:24.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=44346' 2026-02-19T20:36:24.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:24.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=222, tag_tid=14, entry_tid=44346' 2026-02-19T20:36:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44346 2026-02-19T20:36:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 44346 2026-02-19T20:36:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:24.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:24.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:24.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:24.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:24.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:24.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:24.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:24.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:24.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:24.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:24.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:24.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:24.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:24.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:24.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:24.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:24.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:24.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:24.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:24.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:24.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:24.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:24.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:24.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:24.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=44553], [object_number=220, tag_tid=14, entry_tid=44552], [object_number=223, tag_tid=14, entry_tid=44551], [object_number=222, tag_tid=14, entry_tid=44550]]], state=connected] 2026-02-19T20:36:24.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:24.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=221, tag_tid=14, entry_tid=44553], [object_number=220, tag_tid=14, entry_tid=44552], [object_number=223, tag_tid=14, entry_tid=44551], [object_number=222, tag_tid=14, entry_tid=44550]]], state=connected] 2026-02-19T20:36:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=221, tag_tid=14, entry_tid=44553' 2026-02-19T20:36:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=221, tag_tid=14, entry_tid=44553' 2026-02-19T20:36:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=44553' '!=' 'object_number=222, tag_tid=14, entry_tid=44346' 2026-02-19T20:36:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=221, tag_tid=14, entry_tid=44553' = 'object_number=222, tag_tid=14, entry_tid=44346' 2026-02-19T20:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=221, tag_tid=14, entry_tid=44553' 2026-02-19T20:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:24.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:24.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=44553' 2026-02-19T20:36:24.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:24.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:24.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=14, entry_tid=44553' 2026-02-19T20:36:24.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:24.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:24.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44553 2026-02-19T20:36:24.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 44553 2026-02-19T20:36:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:25.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:25.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:25.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:25.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:25.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:25.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:25.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:25.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:25.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=44811], [object_number=222, tag_tid=14, entry_tid=44810], [object_number=221, tag_tid=14, entry_tid=44809], [object_number=220, tag_tid=14, entry_tid=44808]]], state=connected] 2026-02-19T20:36:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:25.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:25.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:25.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:25.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:25.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=44811], [object_number=222, tag_tid=14, entry_tid=44810], [object_number=221, tag_tid=14, entry_tid=44809], [object_number=220, tag_tid=14, entry_tid=44808]]], state=connected] 2026-02-19T20:36:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=223, tag_tid=14, entry_tid=44811' 2026-02-19T20:36:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=223, tag_tid=14, entry_tid=44811' 2026-02-19T20:36:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=44811' '!=' 'object_number=221, tag_tid=14, entry_tid=44553' 2026-02-19T20:36:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=44811' = 'object_number=221, tag_tid=14, entry_tid=44553' 2026-02-19T20:36:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=223, tag_tid=14, entry_tid=44811' 2026-02-19T20:36:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=44811' 2026-02-19T20:36:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:25.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:25.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:25.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:25.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=44811' 2026-02-19T20:36:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44811 2026-02-19T20:36:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 44811 2026-02-19T20:36:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:25.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:25.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:25.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:25.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:25.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:25.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:25.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:25.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:25.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:25.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:25.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:25.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:25.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:25.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:25.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:25.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:25.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:25.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:25.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:25.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:25.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:25.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:25.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=45099], [object_number=222, tag_tid=14, entry_tid=45098], [object_number=221, tag_tid=14, entry_tid=45097], [object_number=220, tag_tid=14, entry_tid=45096]]], state=connected] 2026-02-19T20:36:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:25.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:25.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:25.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:25.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:25.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:25.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:25.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:25.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:25.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:25.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:25.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:25.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:25.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:25.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=45099], [object_number=222, tag_tid=14, entry_tid=45098], [object_number=221, tag_tid=14, entry_tid=45097], [object_number=220, tag_tid=14, entry_tid=45096]]], state=connected] 2026-02-19T20:36:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=223, tag_tid=14, entry_tid=45099' 2026-02-19T20:36:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=223, tag_tid=14, entry_tid=45099' 2026-02-19T20:36:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=45099' '!=' 'object_number=223, tag_tid=14, entry_tid=44811' 2026-02-19T20:36:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=45099' = 'object_number=223, tag_tid=14, entry_tid=44811' 2026-02-19T20:36:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=223, tag_tid=14, entry_tid=45099' 2026-02-19T20:36:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=45099' 2026-02-19T20:36:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=45099' 2026-02-19T20:36:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:25.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45099 2026-02-19T20:36:25.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 45099 2026-02-19T20:36:25.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:25.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:25.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:26.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:26.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:26.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:26.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:26.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:26.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:26.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:26.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198], [object_number=221, tag_tid=14, entry_tid=45197], [object_number=220, tag_tid=14, entry_tid=45196]]], state=connected] 2026-02-19T20:36:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:26.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:26.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:26.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:26.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:26.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:26.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198], [object_number=221, tag_tid=14, entry_tid=45197], [object_number=220, tag_tid=14, entry_tid=45196]]], state=connected] 2026-02-19T20:36:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=223, tag_tid=14, entry_tid=45199' 2026-02-19T20:36:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=223, tag_tid=14, entry_tid=45199' 2026-02-19T20:36:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=45199' '!=' 'object_number=223, tag_tid=14, entry_tid=45099' 2026-02-19T20:36:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:36:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=45199' = 'object_number=223, tag_tid=14, entry_tid=45099' 2026-02-19T20:36:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=223, tag_tid=14, entry_tid=45199' 2026-02-19T20:36:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=15 2026-02-19T20:36:26.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=45199' 2026-02-19T20:36:26.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:36:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=14 2026-02-19T20:36:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:36:26.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=223, tag_tid=14, entry_tid=45199' 2026-02-19T20:36:26.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:36:26.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:36:26.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45199 2026-02-19T20:36:26.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 15 = 14 -a 1 -le 45199 2026-02-19T20:36:26.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:36:26.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:26.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:36:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:26.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:26.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:26.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:26.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:26.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:26.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:26.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:26.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:26.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:26.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:26.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:26.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:26.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:26.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:26.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:26.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:26.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:26.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:26.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:26.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:26.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:26.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:26.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:26.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:26.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:26.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:26.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:26.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:26.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:26.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:26.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:26.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:26.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:26.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:26.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:26.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:26.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:26.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:26.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:26.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:26.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:26.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:26.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:26.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:26.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198], [object_number=221, tag_tid=14, entry_tid=45197], [object_number=220, tag_tid=14, entry_tid=45196]]], state=connected] 2026-02-19T20:36:26.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:26.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:26.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:26.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198], [object_number=221, tag_tid=14, entry_tid=45197], [object_number=220, tag_tid=14, entry_tid=45196]]], state=connected] 2026-02-19T20:36:26.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:26.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=223, tag_tid=14, entry_tid=45199' 2026-02-19T20:36:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=223, tag_tid=14, entry_tid=45199' 2026-02-19T20:36:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=223, tag_tid=14, entry_tid=45199' '!=' 'object_number=223, tag_tid=14, entry_tid=45199' 2026-02-19T20:36:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:36:26.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:36:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:36:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:36:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:36:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:36:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:36:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:36:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:36:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:36:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:36:27.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:36:27.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:36:27.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:27.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:36:27.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:36:27.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:36:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:36:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:36:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:36:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:36:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:36:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:36:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:36:27.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:36:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:36:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:36:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:36:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:36:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:36:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:36:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:36:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:36:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:36:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:36:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:36:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:36:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:36:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:36:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:36:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:36:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:36:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:36:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:36:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:36:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:36:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:36:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:36:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:36:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:36:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:36:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:27.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:27.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:27.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:36:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:36:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:36:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:36:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:36:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:36:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:36:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:36:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:36:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:36:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 55 2026-02-19T20:36:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 55 2026-02-19T20:36:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:36:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=221, tag_tid=15, entry_tid=1], [object_number=220, tag_tid=15, entry_tid=0], [object_number=223, tag_tid=14, entry_tid=45199], [object_number=222, tag_tid=14, entry_tid=45198]]], state=connected] 2026-02-19T20:36:27.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:36:27.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:27.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=221, tag_tid=15, entry_tid=1' -a 'object_number=221, tag_tid=15, entry_tid=1' = 'object_number=221, tag_tid=15, entry_tid=1' 2026-02-19T20:36:27.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:27.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2026-02-19T20:36:27.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap5 2026-02-19T20:36:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:36:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:36:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:36:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2026-02-19T20:36:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:36:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:36:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:36:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 9 2026-02-19T20:36:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:36:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:35:12 2026 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:35:25 2026 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:36:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:36:28.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:36:28.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n snap4-clean ']' 2026-02-19T20:36:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-19T20:36:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap4-clean 2026-02-19T20:36:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:36:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:36:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:36:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:36:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:36:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:36:28.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:36:29.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:36:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:36:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:36:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap5 2026-02-19T20:36:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap5 2026-02-19T20:36:29.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:36:29.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:36:29.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:36:29.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:36:29.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:36:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:36:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:36:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:36:30.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:36:31.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:36:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name=snap5-clean 2026-02-19T20:36:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap5-clean 2026-02-19T20:36:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:36:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap5-clean 2026-02-19T20:36:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2026-02-19T20:36:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:36:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror test 2026-02-19T20:36:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:36:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:36:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=8 2026-02-19T20:36:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:36:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-19T20:36:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 44354560 bytes 2026-02-19T20:36:32.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 46025728 bytes 2026-02-19T20:36:33.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 71501824 bytes 2026-02-19T20:36:34.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 153240576 bytes 2026-02-19T20:36:35.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 218446848 bytes 2026-02-19T20:36:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 219045888 bytes 2026-02-19T20:36:37.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 286910464 bytes 2026-02-19T20:36:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 287537152 bytes 2026-02-19T20:36:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 343220224 bytes 2026-02-19T20:36:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 380481536 bytes 2026-02-19T20:36:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 439437312 bytes 2026-02-19T20:36:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:36:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:36:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:36:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-19T20:36:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:36:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:36:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:36:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:36:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:36:40.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-19T20:36:40.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:36:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:36:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:36:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:36:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:36:40.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='test: 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: global_id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+replaying 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: replaying, {"bytes_per_second":10664423.47,"entries_behind_primary":5269,"entries_per_second":665.0700000000001,"non_primary_position":{"entry_tid":39931,"object_number":219,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":221,"tag_tid":15},"seconds_until_synced":7} 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: service: mirror.1 on trial151 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:36:17 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: peer_sites: 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: name: cluster2 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+stopped 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: local image is primary 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:36:18' 2026-02-19T20:36:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ state: .*up+replaying' 2026-02-19T20:36:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: state: up+replaying 2026-02-19T20:36:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ description: .*' 2026-02-19T20:36:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: description: replaying, {"bytes_per_second":10664423.47,"entries_behind_primary":5269,"entries_per_second":665.0700000000001,"non_primary_position":{"entry_tid":39931,"object_number":219,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":221,"tag_tid":15},"seconds_until_synced":7} 2026-02-19T20:36:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:36:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:36:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^up+' 2026-02-19T20:36:40.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:36:40.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep 'service: *mirror..* on ' 2026-02-19T20:36:40.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: service: mirror.1 on trial151 2026-02-19T20:36:40.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update 2026-02-19T20:36:40.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-19T20:36:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:36:17' 2026-02-19T20:36:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-19 20:36:17' 2026-02-19T20:36:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:36:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:36:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'prev_last_update=2026-02-19 20:36:17' 2026-02-19T20:36:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:36:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:36:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-19T20:36:40.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='OKOKOK142064262mirror.1trial15120.3.0-5336-g802bf111falseOK42104271mirror.2trial15120.3.0-5336-g802bf111trueOK43714374mirror.3trial15120.3.0-5336-g802bf111falseOK44004403mirror.0trial15120.3.0-5336-g802bf111falseOKtest959deb13-de1a-48d9-a341-1f95416f8379up+replayingreplaying, {"bytes_per_second":10664423.47,"entries_behind_primary":5269,"entries_per_second":665.0700000000001,"non_primary_position":{"entry_tid":39931,"object_number":219,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":221,"tag_tid":15},"seconds_until_synced":7}42064262mirror.1trial1512026-02-19 20:36:17cluster2a054b997-3cfb-48e6-978a-70d6a85e56deup+stoppedlocal image is primary2026-02-19 20:36:18' 2026-02-19T20:36:40.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update state 2026-02-19T20:36:40.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-19T20:36:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:36:17' 2026-02-19T20:36:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-19T20:36:40.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=up+replaying 2026-02-19T20:36:40.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:36:40.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep up+replaying 2026-02-19T20:36:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:36:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:36:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap6 2026-02-19T20:36:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap6 2026-02-19T20:36:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:36:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:36:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:36:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap6 2026-02-19T20:36:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:37:08.533 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T20:37:08.533 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T20:37:08.533 INFO:tasks.rbd_mirror_thrash:waiting for 2.6368689400436853 secs before reviving daemons 2026-02-19T20:37:08.534 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T20:37:08.528+0000 7fec32364640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 46684) UID: 1000 2026-02-19T20:37:11.171 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T20:37:11.171 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:37:11.171 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T20:37:11.171 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T20:37:11.171 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T20:37:11.172 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T20:37:11.172 INFO:tasks.rbd_mirror_thrash:waiting for 97.50233775697063 secs before thrashing 2026-02-19T20:37:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:37:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:37:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:37:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:37:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:37:12.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:12.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:37:12.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:37:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:12.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:12.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:12.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:12.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:37:12.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:12.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:37:12.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:37:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:37:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:37:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:37:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:37:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:12.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:12.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:12.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:12.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:37:12.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:37:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:37:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:37:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:37:12.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:37:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:37:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:37:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:37:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:37:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:37:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:37:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:37:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:37:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:37:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:37:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:37:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:37:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:37:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:37:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:37:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:13.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:13.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:13.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:13.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:13.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:13.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:13.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:13.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:13.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:13.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:13.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:13.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 60 2026-02-19T20:37:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=242, tag_tid=17, entry_tid=22806], [object_number=241, tag_tid=17, entry_tid=22805], [object_number=240, tag_tid=17, entry_tid=22804], [object_number=243, tag_tid=17, entry_tid=22803]]], state=connected] 2026-02-19T20:37:13.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 60 2026-02-19T20:37:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=242, tag_tid=17, entry_tid=22806], [object_number=241, tag_tid=17, entry_tid=22805], [object_number=240, tag_tid=17, entry_tid=22804], [object_number=243, tag_tid=17, entry_tid=22803]]], state=connected] 2026-02-19T20:37:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=242, tag_tid=17, entry_tid=22806' 2026-02-19T20:37:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=242, tag_tid=17, entry_tid=22806' 2026-02-19T20:37:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=242, tag_tid=17, entry_tid=22806' '!=' '' 2026-02-19T20:37:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=242, tag_tid=17, entry_tid=22806' = '' 2026-02-19T20:37:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=242, tag_tid=17, entry_tid=22806' 2026-02-19T20:37:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=242, tag_tid=17, entry_tid=22806' 2026-02-19T20:37:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=242, tag_tid=17, entry_tid=22806' 2026-02-19T20:37:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=22806 2026-02-19T20:37:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 22806 2026-02-19T20:37:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:13.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:13.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:13.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:13.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:13.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:13.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:13.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:13.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:13.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:13.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 60 2026-02-19T20:37:13.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:13.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:13.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:13.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=242, tag_tid=17, entry_tid=23078], [object_number=241, tag_tid=17, entry_tid=23077], [object_number=240, tag_tid=17, entry_tid=23076], [object_number=243, tag_tid=17, entry_tid=23075]]], state=connected] 2026-02-19T20:37:13.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 60 2026-02-19T20:37:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=242, tag_tid=17, entry_tid=23078], [object_number=241, tag_tid=17, entry_tid=23077], [object_number=240, tag_tid=17, entry_tid=23076], [object_number=243, tag_tid=17, entry_tid=23075]]], state=connected] 2026-02-19T20:37:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:13.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=242, tag_tid=17, entry_tid=23078' 2026-02-19T20:37:13.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=242, tag_tid=17, entry_tid=23078' 2026-02-19T20:37:13.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=242, tag_tid=17, entry_tid=23078' '!=' 'object_number=242, tag_tid=17, entry_tid=22806' 2026-02-19T20:37:13.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=242, tag_tid=17, entry_tid=23078' = 'object_number=242, tag_tid=17, entry_tid=22806' 2026-02-19T20:37:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=242, tag_tid=17, entry_tid=23078' 2026-02-19T20:37:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:13.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=242, tag_tid=17, entry_tid=23078' 2026-02-19T20:37:13.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:13.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:13.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=242, tag_tid=17, entry_tid=23078' 2026-02-19T20:37:13.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:13.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23078 2026-02-19T20:37:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 23078 2026-02-19T20:37:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:13.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:13.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:13.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:13.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:13.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:13.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:14.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:14.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:14.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:14.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:14.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:14.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:14.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=247, tag_tid=17, entry_tid=23263], [object_number=246, tag_tid=17, entry_tid=23262], [object_number=245, tag_tid=17, entry_tid=23261], [object_number=244, tag_tid=17, entry_tid=23260]]], state=connected] 2026-02-19T20:37:14.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=247, tag_tid=17, entry_tid=23263], [object_number=246, tag_tid=17, entry_tid=23262], [object_number=245, tag_tid=17, entry_tid=23261], [object_number=244, tag_tid=17, entry_tid=23260]]], state=connected] 2026-02-19T20:37:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=247, tag_tid=17, entry_tid=23263' 2026-02-19T20:37:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=247, tag_tid=17, entry_tid=23263' 2026-02-19T20:37:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=247, tag_tid=17, entry_tid=23263' '!=' 'object_number=242, tag_tid=17, entry_tid=23078' 2026-02-19T20:37:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:14.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=247, tag_tid=17, entry_tid=23263' = 'object_number=242, tag_tid=17, entry_tid=23078' 2026-02-19T20:37:14.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=247, tag_tid=17, entry_tid=23263' 2026-02-19T20:37:14.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=247, tag_tid=17, entry_tid=23263' 2026-02-19T20:37:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:14.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:14.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:14.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=247, tag_tid=17, entry_tid=23263' 2026-02-19T20:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23263 2026-02-19T20:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 23263 2026-02-19T20:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:14.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:14.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:14.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:14.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:14.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:14.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:14.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:14.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:14.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:14.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:14.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:14.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:14.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=247, tag_tid=17, entry_tid=23547], [object_number=246, tag_tid=17, entry_tid=23546], [object_number=245, tag_tid=17, entry_tid=23545], [object_number=244, tag_tid=17, entry_tid=23544]]], state=connected] 2026-02-19T20:37:15.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=247, tag_tid=17, entry_tid=23547], [object_number=246, tag_tid=17, entry_tid=23546], [object_number=245, tag_tid=17, entry_tid=23545], [object_number=244, tag_tid=17, entry_tid=23544]]], state=connected] 2026-02-19T20:37:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=247, tag_tid=17, entry_tid=23547' 2026-02-19T20:37:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=247, tag_tid=17, entry_tid=23547' 2026-02-19T20:37:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=247, tag_tid=17, entry_tid=23547' '!=' 'object_number=247, tag_tid=17, entry_tid=23263' 2026-02-19T20:37:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=247, tag_tid=17, entry_tid=23547' = 'object_number=247, tag_tid=17, entry_tid=23263' 2026-02-19T20:37:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=247, tag_tid=17, entry_tid=23547' 2026-02-19T20:37:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=247, tag_tid=17, entry_tid=23547' 2026-02-19T20:37:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=247, tag_tid=17, entry_tid=23547' 2026-02-19T20:37:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:15.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23547 2026-02-19T20:37:15.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 23547 2026-02-19T20:37:15.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:15.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:15.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:15.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:15.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:15.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:15.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:15.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:15.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:15.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:15.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:15.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:15.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:15.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:15.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:15.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:15.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:15.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:15.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:15.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=245, tag_tid=17, entry_tid=24257], [object_number=244, tag_tid=17, entry_tid=24256], [object_number=247, tag_tid=17, entry_tid=24255], [object_number=246, tag_tid=17, entry_tid=24254]]], state=connected] 2026-02-19T20:37:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:15.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:15.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=245, tag_tid=17, entry_tid=24257], [object_number=244, tag_tid=17, entry_tid=24256], [object_number=247, tag_tid=17, entry_tid=24255], [object_number=246, tag_tid=17, entry_tid=24254]]], state=connected] 2026-02-19T20:37:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=245, tag_tid=17, entry_tid=24257' 2026-02-19T20:37:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=245, tag_tid=17, entry_tid=24257' 2026-02-19T20:37:15.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=245, tag_tid=17, entry_tid=24257' '!=' 'object_number=247, tag_tid=17, entry_tid=23547' 2026-02-19T20:37:15.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:15.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=245, tag_tid=17, entry_tid=24257' = 'object_number=247, tag_tid=17, entry_tid=23547' 2026-02-19T20:37:15.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=245, tag_tid=17, entry_tid=24257' 2026-02-19T20:37:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=245, tag_tid=17, entry_tid=24257' 2026-02-19T20:37:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:15.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:15.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:15.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=245, tag_tid=17, entry_tid=24257' 2026-02-19T20:37:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24257 2026-02-19T20:37:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 24257 2026-02-19T20:37:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:15.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:15.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:16.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:16.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:16.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=244, tag_tid=17, entry_tid=24600], [object_number=247, tag_tid=17, entry_tid=24599], [object_number=246, tag_tid=17, entry_tid=24598], [object_number=245, tag_tid=17, entry_tid=24597]]], state=connected] 2026-02-19T20:37:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=244, tag_tid=17, entry_tid=24600], [object_number=247, tag_tid=17, entry_tid=24599], [object_number=246, tag_tid=17, entry_tid=24598], [object_number=245, tag_tid=17, entry_tid=24597]]], state=connected] 2026-02-19T20:37:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=244, tag_tid=17, entry_tid=24600' 2026-02-19T20:37:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=244, tag_tid=17, entry_tid=24600' 2026-02-19T20:37:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=244, tag_tid=17, entry_tid=24600' '!=' 'object_number=245, tag_tid=17, entry_tid=24257' 2026-02-19T20:37:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=244, tag_tid=17, entry_tid=24600' = 'object_number=245, tag_tid=17, entry_tid=24257' 2026-02-19T20:37:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=244, tag_tid=17, entry_tid=24600' 2026-02-19T20:37:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=244, tag_tid=17, entry_tid=24600' 2026-02-19T20:37:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:16.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:16.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=244, tag_tid=17, entry_tid=24600' 2026-02-19T20:37:16.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:16.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24600 2026-02-19T20:37:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 24600 2026-02-19T20:37:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:16.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:16.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:16.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:16.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:16.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:16.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:16.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:16.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=247, tag_tid=17, entry_tid=24867], [object_number=246, tag_tid=17, entry_tid=24866], [object_number=245, tag_tid=17, entry_tid=24865], [object_number=244, tag_tid=17, entry_tid=24864]]], state=connected] 2026-02-19T20:37:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:16.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:16.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=247, tag_tid=17, entry_tid=24867], [object_number=246, tag_tid=17, entry_tid=24866], [object_number=245, tag_tid=17, entry_tid=24865], [object_number=244, tag_tid=17, entry_tid=24864]]], state=connected] 2026-02-19T20:37:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=247, tag_tid=17, entry_tid=24867' 2026-02-19T20:37:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=247, tag_tid=17, entry_tid=24867' 2026-02-19T20:37:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=247, tag_tid=17, entry_tid=24867' '!=' 'object_number=244, tag_tid=17, entry_tid=24600' 2026-02-19T20:37:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=247, tag_tid=17, entry_tid=24867' = 'object_number=244, tag_tid=17, entry_tid=24600' 2026-02-19T20:37:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=247, tag_tid=17, entry_tid=24867' 2026-02-19T20:37:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:16.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=247, tag_tid=17, entry_tid=24867' 2026-02-19T20:37:16.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:16.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:16.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:16.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:16.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:16.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=247, tag_tid=17, entry_tid=24867' 2026-02-19T20:37:16.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:16.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:16.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24867 2026-02-19T20:37:16.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 24867 2026-02-19T20:37:16.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:16.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:16.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:16.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:16.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:17.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:17.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=245, tag_tid=17, entry_tid=25253], [object_number=244, tag_tid=17, entry_tid=25252], [object_number=247, tag_tid=17, entry_tid=25251], [object_number=246, tag_tid=17, entry_tid=25250]]], state=connected] 2026-02-19T20:37:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:17.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=245, tag_tid=17, entry_tid=25253], [object_number=244, tag_tid=17, entry_tid=25252], [object_number=247, tag_tid=17, entry_tid=25251], [object_number=246, tag_tid=17, entry_tid=25250]]], state=connected] 2026-02-19T20:37:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=245, tag_tid=17, entry_tid=25253' 2026-02-19T20:37:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=245, tag_tid=17, entry_tid=25253' 2026-02-19T20:37:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=245, tag_tid=17, entry_tid=25253' '!=' 'object_number=247, tag_tid=17, entry_tid=24867' 2026-02-19T20:37:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=245, tag_tid=17, entry_tid=25253' = 'object_number=247, tag_tid=17, entry_tid=24867' 2026-02-19T20:37:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=245, tag_tid=17, entry_tid=25253' 2026-02-19T20:37:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=245, tag_tid=17, entry_tid=25253' 2026-02-19T20:37:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=245, tag_tid=17, entry_tid=25253' 2026-02-19T20:37:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25253 2026-02-19T20:37:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 25253 2026-02-19T20:37:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:17.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:17.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:17.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:17.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:17.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:17.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:17.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:17.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:17.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:17.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:17.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:17.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=246, tag_tid=17, entry_tid=25522], [object_number=245, tag_tid=17, entry_tid=25521], [object_number=244, tag_tid=17, entry_tid=25520], [object_number=247, tag_tid=17, entry_tid=25519]]], state=connected] 2026-02-19T20:37:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:17.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:17.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:17.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:17.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:17.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=246, tag_tid=17, entry_tid=25522], [object_number=245, tag_tid=17, entry_tid=25521], [object_number=244, tag_tid=17, entry_tid=25520], [object_number=247, tag_tid=17, entry_tid=25519]]], state=connected] 2026-02-19T20:37:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=246, tag_tid=17, entry_tid=25522' 2026-02-19T20:37:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=246, tag_tid=17, entry_tid=25522' 2026-02-19T20:37:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=246, tag_tid=17, entry_tid=25522' '!=' 'object_number=245, tag_tid=17, entry_tid=25253' 2026-02-19T20:37:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=246, tag_tid=17, entry_tid=25522' = 'object_number=245, tag_tid=17, entry_tid=25253' 2026-02-19T20:37:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=246, tag_tid=17, entry_tid=25522' 2026-02-19T20:37:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=246, tag_tid=17, entry_tid=25522' 2026-02-19T20:37:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:17.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=246, tag_tid=17, entry_tid=25522' 2026-02-19T20:37:17.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:17.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:17.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25522 2026-02-19T20:37:17.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 25522 2026-02-19T20:37:17.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:17.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:17.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:17.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:17.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:17.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:17.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:17.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=246, tag_tid=17, entry_tid=25790], [object_number=245, tag_tid=17, entry_tid=25789], [object_number=244, tag_tid=17, entry_tid=25788], [object_number=247, tag_tid=17, entry_tid=25787]]], state=connected] 2026-02-19T20:37:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:17.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=246, tag_tid=17, entry_tid=25790], [object_number=245, tag_tid=17, entry_tid=25789], [object_number=244, tag_tid=17, entry_tid=25788], [object_number=247, tag_tid=17, entry_tid=25787]]], state=connected] 2026-02-19T20:37:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=246, tag_tid=17, entry_tid=25790' 2026-02-19T20:37:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=246, tag_tid=17, entry_tid=25790' 2026-02-19T20:37:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=246, tag_tid=17, entry_tid=25790' '!=' 'object_number=246, tag_tid=17, entry_tid=25522' 2026-02-19T20:37:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=246, tag_tid=17, entry_tid=25790' = 'object_number=246, tag_tid=17, entry_tid=25522' 2026-02-19T20:37:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=246, tag_tid=17, entry_tid=25790' 2026-02-19T20:37:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=246, tag_tid=17, entry_tid=25790' 2026-02-19T20:37:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=246, tag_tid=17, entry_tid=25790' 2026-02-19T20:37:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25790 2026-02-19T20:37:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 25790 2026-02-19T20:37:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:18.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:18.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:18.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:18.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:18.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:18.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:18.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:18.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:18.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:18.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:18.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=247, tag_tid=17, entry_tid=26119], [object_number=246, tag_tid=17, entry_tid=26118], [object_number=245, tag_tid=17, entry_tid=26117], [object_number=244, tag_tid=17, entry_tid=26116]]], state=connected] 2026-02-19T20:37:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:18.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:18.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:18.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:18.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:18.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:18.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:18.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:18.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:18.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:18.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:18.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:18.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:18.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:18.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:18.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:18.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=247, tag_tid=17, entry_tid=26119], [object_number=246, tag_tid=17, entry_tid=26118], [object_number=245, tag_tid=17, entry_tid=26117], [object_number=244, tag_tid=17, entry_tid=26116]]], state=connected] 2026-02-19T20:37:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=247, tag_tid=17, entry_tid=26119' 2026-02-19T20:37:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=247, tag_tid=17, entry_tid=26119' 2026-02-19T20:37:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=247, tag_tid=17, entry_tid=26119' '!=' 'object_number=246, tag_tid=17, entry_tid=25790' 2026-02-19T20:37:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=247, tag_tid=17, entry_tid=26119' = 'object_number=246, tag_tid=17, entry_tid=25790' 2026-02-19T20:37:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=247, tag_tid=17, entry_tid=26119' 2026-02-19T20:37:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:18.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:18.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=247, tag_tid=17, entry_tid=26119' 2026-02-19T20:37:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:18.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:18.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:18.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:18.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:18.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:18.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=247, tag_tid=17, entry_tid=26119' 2026-02-19T20:37:18.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:18.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:18.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26119 2026-02-19T20:37:18.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 26119 2026-02-19T20:37:18.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:18.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:18.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:18.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:18.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=245, tag_tid=17, entry_tid=26529], [object_number=244, tag_tid=17, entry_tid=26528], [object_number=247, tag_tid=17, entry_tid=26527], [object_number=246, tag_tid=17, entry_tid=26526]]], state=connected] 2026-02-19T20:37:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=245, tag_tid=17, entry_tid=26529], [object_number=244, tag_tid=17, entry_tid=26528], [object_number=247, tag_tid=17, entry_tid=26527], [object_number=246, tag_tid=17, entry_tid=26526]]], state=connected] 2026-02-19T20:37:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=245, tag_tid=17, entry_tid=26529' 2026-02-19T20:37:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=245, tag_tid=17, entry_tid=26529' 2026-02-19T20:37:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=245, tag_tid=17, entry_tid=26529' '!=' 'object_number=247, tag_tid=17, entry_tid=26119' 2026-02-19T20:37:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=245, tag_tid=17, entry_tid=26529' = 'object_number=247, tag_tid=17, entry_tid=26119' 2026-02-19T20:37:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=245, tag_tid=17, entry_tid=26529' 2026-02-19T20:37:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=245, tag_tid=17, entry_tid=26529' 2026-02-19T20:37:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=245, tag_tid=17, entry_tid=26529' 2026-02-19T20:37:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26529 2026-02-19T20:37:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 26529 2026-02-19T20:37:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:19.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:19.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:19.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:19.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:19.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=244, tag_tid=17, entry_tid=26808], [object_number=247, tag_tid=17, entry_tid=26807], [object_number=246, tag_tid=17, entry_tid=26806], [object_number=245, tag_tid=17, entry_tid=26805]]], state=connected] 2026-02-19T20:37:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:19.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:19.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:19.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=244, tag_tid=17, entry_tid=26808], [object_number=247, tag_tid=17, entry_tid=26807], [object_number=246, tag_tid=17, entry_tid=26806], [object_number=245, tag_tid=17, entry_tid=26805]]], state=connected] 2026-02-19T20:37:19.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=244, tag_tid=17, entry_tid=26808' 2026-02-19T20:37:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=244, tag_tid=17, entry_tid=26808' 2026-02-19T20:37:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=244, tag_tid=17, entry_tid=26808' '!=' 'object_number=245, tag_tid=17, entry_tid=26529' 2026-02-19T20:37:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=244, tag_tid=17, entry_tid=26808' = 'object_number=245, tag_tid=17, entry_tid=26529' 2026-02-19T20:37:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=244, tag_tid=17, entry_tid=26808' 2026-02-19T20:37:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=244, tag_tid=17, entry_tid=26808' 2026-02-19T20:37:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=244, tag_tid=17, entry_tid=26808' 2026-02-19T20:37:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26808 2026-02-19T20:37:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 26808 2026-02-19T20:37:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:19.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:19.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:19.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:19.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:19.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:19.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:19.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:19.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:19.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:19.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:19.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:19.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:19.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:19.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:19.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:19.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:19.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:19.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:19.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:19.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:19.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=244, tag_tid=17, entry_tid=27092], [object_number=247, tag_tid=17, entry_tid=27091], [object_number=246, tag_tid=17, entry_tid=27090], [object_number=245, tag_tid=17, entry_tid=27089]]], state=connected] 2026-02-19T20:37:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=244, tag_tid=17, entry_tid=27092], [object_number=247, tag_tid=17, entry_tid=27091], [object_number=246, tag_tid=17, entry_tid=27090], [object_number=245, tag_tid=17, entry_tid=27089]]], state=connected] 2026-02-19T20:37:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=244, tag_tid=17, entry_tid=27092' 2026-02-19T20:37:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=244, tag_tid=17, entry_tid=27092' 2026-02-19T20:37:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=244, tag_tid=17, entry_tid=27092' '!=' 'object_number=244, tag_tid=17, entry_tid=26808' 2026-02-19T20:37:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=244, tag_tid=17, entry_tid=27092' = 'object_number=244, tag_tid=17, entry_tid=26808' 2026-02-19T20:37:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=244, tag_tid=17, entry_tid=27092' 2026-02-19T20:37:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=244, tag_tid=17, entry_tid=27092' 2026-02-19T20:37:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=244, tag_tid=17, entry_tid=27092' 2026-02-19T20:37:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27092 2026-02-19T20:37:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 27092 2026-02-19T20:37:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:20.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=245, tag_tid=17, entry_tid=27349], [object_number=244, tag_tid=17, entry_tid=27348], [object_number=247, tag_tid=17, entry_tid=27347], [object_number=246, tag_tid=17, entry_tid=27346]]], state=connected] 2026-02-19T20:37:20.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:20.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:20.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 61 2026-02-19T20:37:20.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:20.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:20.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:20.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=245, tag_tid=17, entry_tid=27349], [object_number=244, tag_tid=17, entry_tid=27348], [object_number=247, tag_tid=17, entry_tid=27347], [object_number=246, tag_tid=17, entry_tid=27346]]], state=connected] 2026-02-19T20:37:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=245, tag_tid=17, entry_tid=27349' 2026-02-19T20:37:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=245, tag_tid=17, entry_tid=27349' 2026-02-19T20:37:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=245, tag_tid=17, entry_tid=27349' '!=' 'object_number=244, tag_tid=17, entry_tid=27092' 2026-02-19T20:37:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=245, tag_tid=17, entry_tid=27349' = 'object_number=244, tag_tid=17, entry_tid=27092' 2026-02-19T20:37:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=245, tag_tid=17, entry_tid=27349' 2026-02-19T20:37:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:20.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=245, tag_tid=17, entry_tid=27349' 2026-02-19T20:37:20.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:20.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:20.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=245, tag_tid=17, entry_tid=27349' 2026-02-19T20:37:20.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:20.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27349 2026-02-19T20:37:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 27349 2026-02-19T20:37:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:20.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:20.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:20.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:20.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:20.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=251, tag_tid=17, entry_tid=27583], [object_number=250, tag_tid=17, entry_tid=27582], [object_number=249, tag_tid=17, entry_tid=27581], [object_number=248, tag_tid=17, entry_tid=27580]]], state=connected] 2026-02-19T20:37:20.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:20.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:20.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:20.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:20.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:20.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:20.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=251, tag_tid=17, entry_tid=27583], [object_number=250, tag_tid=17, entry_tid=27582], [object_number=249, tag_tid=17, entry_tid=27581], [object_number=248, tag_tid=17, entry_tid=27580]]], state=connected] 2026-02-19T20:37:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=251, tag_tid=17, entry_tid=27583' 2026-02-19T20:37:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=251, tag_tid=17, entry_tid=27583' 2026-02-19T20:37:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=251, tag_tid=17, entry_tid=27583' '!=' 'object_number=245, tag_tid=17, entry_tid=27349' 2026-02-19T20:37:20.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:20.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=251, tag_tid=17, entry_tid=27583' = 'object_number=245, tag_tid=17, entry_tid=27349' 2026-02-19T20:37:20.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=251, tag_tid=17, entry_tid=27583' 2026-02-19T20:37:20.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=251, tag_tid=17, entry_tid=27583' 2026-02-19T20:37:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:20.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:20.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:20.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:20.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:20.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:20.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:20.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=251, tag_tid=17, entry_tid=27583' 2026-02-19T20:37:20.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:20.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:20.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27583 2026-02-19T20:37:20.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 27583 2026-02-19T20:37:20.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:20.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:20.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:21.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:21.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:21.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:21.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:21.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:21.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:21.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:21.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=27866], [object_number=249, tag_tid=17, entry_tid=27865], [object_number=248, tag_tid=17, entry_tid=27864], [object_number=251, tag_tid=17, entry_tid=27863]]], state=connected] 2026-02-19T20:37:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:21.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:21.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:21.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:21.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:21.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:21.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:21.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:21.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:21.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:21.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:21.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:21.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:21.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:21.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=27866], [object_number=249, tag_tid=17, entry_tid=27865], [object_number=248, tag_tid=17, entry_tid=27864], [object_number=251, tag_tid=17, entry_tid=27863]]], state=connected] 2026-02-19T20:37:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=250, tag_tid=17, entry_tid=27866' 2026-02-19T20:37:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=250, tag_tid=17, entry_tid=27866' 2026-02-19T20:37:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=27866' '!=' 'object_number=251, tag_tid=17, entry_tid=27583' 2026-02-19T20:37:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=27866' = 'object_number=251, tag_tid=17, entry_tid=27583' 2026-02-19T20:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=250, tag_tid=17, entry_tid=27866' 2026-02-19T20:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=250, tag_tid=17, entry_tid=27866' 2026-02-19T20:37:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:21.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=250, tag_tid=17, entry_tid=27866' 2026-02-19T20:37:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27866 2026-02-19T20:37:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 27866 2026-02-19T20:37:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:21.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:21.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:21.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:21.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:21.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:21.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:21.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:21.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:21.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:21.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:21.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:21.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:21.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:21.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:21.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:21.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:21.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:21.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:21.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:21.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:21.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:21.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:21.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:21.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:21.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:21.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:21.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:21.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:21.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:21.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:21.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:21.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:21.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:21.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:21.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=28142], [object_number=249, tag_tid=17, entry_tid=28141], [object_number=248, tag_tid=17, entry_tid=28140], [object_number=251, tag_tid=17, entry_tid=28139]]], state=connected] 2026-02-19T20:37:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=28142], [object_number=249, tag_tid=17, entry_tid=28141], [object_number=248, tag_tid=17, entry_tid=28140], [object_number=251, tag_tid=17, entry_tid=28139]]], state=connected] 2026-02-19T20:37:21.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=250, tag_tid=17, entry_tid=28142' 2026-02-19T20:37:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=250, tag_tid=17, entry_tid=28142' 2026-02-19T20:37:21.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=28142' '!=' 'object_number=250, tag_tid=17, entry_tid=27866' 2026-02-19T20:37:21.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:21.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=28142' = 'object_number=250, tag_tid=17, entry_tid=27866' 2026-02-19T20:37:21.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=250, tag_tid=17, entry_tid=28142' 2026-02-19T20:37:21.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:21.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:21.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:21.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:21.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=250, tag_tid=17, entry_tid=28142' 2026-02-19T20:37:21.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:21.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=250, tag_tid=17, entry_tid=28142' 2026-02-19T20:37:21.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:21.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28142 2026-02-19T20:37:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 28142 2026-02-19T20:37:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:21.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:21.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:21.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:21.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:21.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:21.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:21.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:21.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:21.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:21.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:21.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=251, tag_tid=17, entry_tid=28431], [object_number=250, tag_tid=17, entry_tid=28430], [object_number=249, tag_tid=17, entry_tid=28429], [object_number=248, tag_tid=17, entry_tid=28428]]], state=connected] 2026-02-19T20:37:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:22.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:22.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=251, tag_tid=17, entry_tid=28431], [object_number=250, tag_tid=17, entry_tid=28430], [object_number=249, tag_tid=17, entry_tid=28429], [object_number=248, tag_tid=17, entry_tid=28428]]], state=connected] 2026-02-19T20:37:22.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=251, tag_tid=17, entry_tid=28431' 2026-02-19T20:37:22.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=251, tag_tid=17, entry_tid=28431' 2026-02-19T20:37:22.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=251, tag_tid=17, entry_tid=28431' '!=' 'object_number=250, tag_tid=17, entry_tid=28142' 2026-02-19T20:37:22.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:22.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=251, tag_tid=17, entry_tid=28431' = 'object_number=250, tag_tid=17, entry_tid=28142' 2026-02-19T20:37:22.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=251, tag_tid=17, entry_tid=28431' 2026-02-19T20:37:22.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=251, tag_tid=17, entry_tid=28431' 2026-02-19T20:37:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=251, tag_tid=17, entry_tid=28431' 2026-02-19T20:37:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28431 2026-02-19T20:37:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 28431 2026-02-19T20:37:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:22.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:22.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:22.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:22.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:22.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:22.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:22.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:22.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:22.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:22.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:22.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:22.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:22.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:22.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=251, tag_tid=17, entry_tid=28711], [object_number=250, tag_tid=17, entry_tid=28710], [object_number=249, tag_tid=17, entry_tid=28709], [object_number=248, tag_tid=17, entry_tid=28708]]], state=connected] 2026-02-19T20:37:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=251, tag_tid=17, entry_tid=28711], [object_number=250, tag_tid=17, entry_tid=28710], [object_number=249, tag_tid=17, entry_tid=28709], [object_number=248, tag_tid=17, entry_tid=28708]]], state=connected] 2026-02-19T20:37:22.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=251, tag_tid=17, entry_tid=28711' 2026-02-19T20:37:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=251, tag_tid=17, entry_tid=28711' 2026-02-19T20:37:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=251, tag_tid=17, entry_tid=28711' '!=' 'object_number=251, tag_tid=17, entry_tid=28431' 2026-02-19T20:37:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=251, tag_tid=17, entry_tid=28711' = 'object_number=251, tag_tid=17, entry_tid=28431' 2026-02-19T20:37:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=251, tag_tid=17, entry_tid=28711' 2026-02-19T20:37:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=251, tag_tid=17, entry_tid=28711' 2026-02-19T20:37:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=251, tag_tid=17, entry_tid=28711' 2026-02-19T20:37:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28711 2026-02-19T20:37:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 28711 2026-02-19T20:37:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:22.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:22.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:22.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:22.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:22.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:22.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:22.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:22.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=28994], [object_number=249, tag_tid=17, entry_tid=28993], [object_number=248, tag_tid=17, entry_tid=28992], [object_number=251, tag_tid=17, entry_tid=28991]]], state=connected] 2026-02-19T20:37:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=28994], [object_number=249, tag_tid=17, entry_tid=28993], [object_number=248, tag_tid=17, entry_tid=28992], [object_number=251, tag_tid=17, entry_tid=28991]]], state=connected] 2026-02-19T20:37:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=250, tag_tid=17, entry_tid=28994' 2026-02-19T20:37:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=250, tag_tid=17, entry_tid=28994' 2026-02-19T20:37:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=28994' '!=' 'object_number=251, tag_tid=17, entry_tid=28711' 2026-02-19T20:37:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=28994' = 'object_number=251, tag_tid=17, entry_tid=28711' 2026-02-19T20:37:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=250, tag_tid=17, entry_tid=28994' 2026-02-19T20:37:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=250, tag_tid=17, entry_tid=28994' 2026-02-19T20:37:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:22.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=250, tag_tid=17, entry_tid=28994' 2026-02-19T20:37:22.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:22.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28994 2026-02-19T20:37:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 28994 2026-02-19T20:37:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:23.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:23.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:23.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:23.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:23.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=29230], [object_number=249, tag_tid=17, entry_tid=29229], [object_number=248, tag_tid=17, entry_tid=29228], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:23.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=29230], [object_number=249, tag_tid=17, entry_tid=29229], [object_number=248, tag_tid=17, entry_tid=29228], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=29230' '!=' 'object_number=250, tag_tid=17, entry_tid=28994' 2026-02-19T20:37:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:37:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=29230' = 'object_number=250, tag_tid=17, entry_tid=28994' 2026-02-19T20:37:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=18 2026-02-19T20:37:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:37:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=17 2026-02-19T20:37:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:37:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:37:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:37:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29230 2026-02-19T20:37:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 18 = 17 -a 1 -le 29230 2026-02-19T20:37:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:37:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:37:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:23.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:23.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:23.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:23.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:23.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:23.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:23.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:23.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=29230], [object_number=249, tag_tid=17, entry_tid=29229], [object_number=248, tag_tid=17, entry_tid=29228], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=29230], [object_number=249, tag_tid=17, entry_tid=29229], [object_number=248, tag_tid=17, entry_tid=29228], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=29230' '!=' 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:37:24.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:24.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:24.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:24.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:24.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:24.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:24.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:24.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:24.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:24.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:24.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:24.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:24.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:24.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:24.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:24.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:24.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:24.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=29230], [object_number=249, tag_tid=17, entry_tid=29229], [object_number=248, tag_tid=17, entry_tid=29228], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:24.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:24.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=29230], [object_number=249, tag_tid=17, entry_tid=29229], [object_number=248, tag_tid=17, entry_tid=29228], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:24.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=29230' '!=' 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:24.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:37:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:25.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:25.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:25.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:25.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:25.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:25.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:25.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:25.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:25.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:25.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:25.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:25.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:25.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:25.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:25.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:25.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:25.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:25.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:25.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:25.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:25.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:25.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:25.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:25.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:25.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=29230], [object_number=249, tag_tid=17, entry_tid=29229], [object_number=248, tag_tid=17, entry_tid=29228], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:25.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=29230], [object_number=249, tag_tid=17, entry_tid=29229], [object_number=248, tag_tid=17, entry_tid=29228], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:25.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:25.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:25.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=29230' '!=' 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:25.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:25.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:37:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:27.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:27.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:27.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:27.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:27.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:27.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:27.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:27.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:27.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:27.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:27.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:27.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:27.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=29230], [object_number=249, tag_tid=17, entry_tid=29229], [object_number=248, tag_tid=17, entry_tid=29228], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:27.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:27.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:27.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:27.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:27.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:27.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:27.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:27.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:27.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:27.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:27.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:27.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:27.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:27.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:27.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:27.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:27.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=250, tag_tid=17, entry_tid=29230], [object_number=249, tag_tid=17, entry_tid=29229], [object_number=248, tag_tid=17, entry_tid=29228], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:27.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=250, tag_tid=17, entry_tid=29230' '!=' 'object_number=250, tag_tid=17, entry_tid=29230' 2026-02-19T20:37:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:37:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:37:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:37:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:37:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:37:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:37:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:37:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:37:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:37:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:37:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:37:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:37:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:37:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:37:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:37:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:37:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:37:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:37:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:37:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:37:29.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:37:29.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:37:29.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:37:29.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:37:29.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:37:29.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:37:29.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:37:29.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:37:29.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:37:29.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:37:29.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:37:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:37:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:37:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:37:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:37:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:37:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:37:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:37:29.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:29.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:29.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:29.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:29.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:29.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:29.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:29.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:29.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:37:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:37:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:37:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:37:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:37:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:37:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:37:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:37:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:37:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:37:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 62 2026-02-19T20:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 62 2026-02-19T20:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=249, tag_tid=18, entry_tid=1], [object_number=248, tag_tid=18, entry_tid=0], [object_number=250, tag_tid=17, entry_tid=29230], [object_number=251, tag_tid=17, entry_tid=29227]]], state=connected] 2026-02-19T20:37:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:37:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=249, tag_tid=18, entry_tid=1' -a 'object_number=249, tag_tid=18, entry_tid=1' = 'object_number=249, tag_tid=18, entry_tid=1' 2026-02-19T20:37:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2026-02-19T20:37:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap6 2026-02-19T20:37:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:37:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:37:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:37:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2026-02-19T20:37:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:37:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:37:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 11 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:36:28 2026 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:36:37 2026 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:37:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n snap5-clean ']' 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap5-clean 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:37:30.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:37:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:37:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:37:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:37:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:37:31.712 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3205689784152478) > weight (0.25) 2026-02-19T20:37:31.712 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9776859422660014) > weight (0.25) 2026-02-19T20:37:31.712 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6509901516529513) > weight (0.25) 2026-02-19T20:37:31.712 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.5507568559758697) > weight (0.25) 2026-02-19T20:37:31.712 INFO:tasks.rbd_mirror_thrash:waiting for 97.36377445800565 secs before thrashing 2026-02-19T20:37:31.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:37:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap6 2026-02-19T20:37:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap6 2026-02-19T20:37:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:37:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:37:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:37:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:37:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:37:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:37:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:37:32.782 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:37:32.774+0000 7f9395725640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-19T20:37:32.782 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:37:32.774+0000 7f9397f2a640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-19T20:37:32.782 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:37:32.774+0000 7f9393f22640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-19T20:37:32.782 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:37:32.774+0000 7f939af30640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-19T20:37:32.783 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:37:32.778+0000 7f939a72f640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-19T20:37:32.783 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:37:32.778+0000 7f9398f2c640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-19T20:37:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:37:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:37:33.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:37:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name=snap6-clean 2026-02-19T20:37:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap6-clean 2026-02-19T20:37:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:37:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap6-clean 2026-02-19T20:37:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2026-02-19T20:37:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:37:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror test 2026-02-19T20:37:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:37:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:37:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=13 2026-02-19T20:37:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:37:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-19T20:37:33.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 631808 bytes 2026-02-19T20:37:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 62531584 bytes 2026-02-19T20:37:35.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 127558656 bytes 2026-02-19T20:37:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 167985152 bytes 2026-02-19T20:37:36.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 189362176 bytes 2026-02-19T20:37:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 220924928 bytes 2026-02-19T20:37:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 249321472 bytes 2026-02-19T20:37:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 284241920 bytes 2026-02-19T20:37:38.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 327606272 bytes 2026-02-19T20:37:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 356402176 bytes 2026-02-19T20:37:39.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 407369728 bytes 2026-02-19T20:37:40.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 444560384 bytes 2026-02-19T20:37:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 455370752 bytes 2026-02-19T20:37:40.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 494068736 bytes 2026-02-19T20:37:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 572293120 bytes 2026-02-19T20:37:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 615307264 bytes 2026-02-19T20:37:42.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 636703744 bytes 2026-02-19T20:37:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 682003456 bytes 2026-02-19T20:37:43.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 727137280 bytes 2026-02-19T20:37:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2000 IOs, 740648960 bytes 2026-02-19T20:37:44.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2100 IOs, 762530816 bytes 2026-02-19T20:37:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2200 IOs, 830110720 bytes 2026-02-19T20:37:46.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2300 IOs, 890178560 bytes 2026-02-19T20:37:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:37:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:37:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:37:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-19T20:37:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:37:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:37:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:37:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:37:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:37:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-19T20:37:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:37:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:37:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:37:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:37:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:37:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-19T20:37:47.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='test: 2026-02-19T20:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: global_id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+replaying 2026-02-19T20:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: replaying, {"bytes_per_second":10420381.87,"entries_behind_primary":3709,"entries_per_second":652.0,"non_primary_position":{"entry_tid":25522,"object_number":246,"tag_tid":17},"primary_position":{"entry_tid":1,"object_number":249,"tag_tid":18},"seconds_until_synced":5} 2026-02-19T20:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: service: mirror.1 on trial151 2026-02-19T20:37:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:37:17 2026-02-19T20:37:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: peer_sites: 2026-02-19T20:37:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: name: cluster2 2026-02-19T20:37:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+stopped 2026-02-19T20:37:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: local image is primary 2026-02-19T20:37:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:37:18' 2026-02-19T20:37:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ state: .*up+replaying' 2026-02-19T20:37:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: state: up+replaying 2026-02-19T20:37:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ description: .*' 2026-02-19T20:37:47.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: description: replaying, {"bytes_per_second":10420381.87,"entries_behind_primary":3709,"entries_per_second":652.0,"non_primary_position":{"entry_tid":25522,"object_number":246,"tag_tid":17},"primary_position":{"entry_tid":1,"object_number":249,"tag_tid":18},"seconds_until_synced":5} 2026-02-19T20:37:47.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:37:47.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:37:47.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^up+' 2026-02-19T20:37:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:37:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep 'service: *mirror..* on ' 2026-02-19T20:37:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: service: mirror.1 on trial151 2026-02-19T20:37:47.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update 2026-02-19T20:37:47.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-19T20:37:47.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:37:17' 2026-02-19T20:37:47.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-19 20:37:17' 2026-02-19T20:37:47.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:37:47.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:47.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:47.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:37:47.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'prev_last_update=2026-02-19 20:37:17' 2026-02-19T20:37:47.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:37:47.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:37:47.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-19T20:37:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='OKOKOK142064262mirror.1trial15120.3.0-5336-g802bf111falseOK42104271mirror.2trial15120.3.0-5336-g802bf111trueOK43714374mirror.3trial15120.3.0-5336-g802bf111falseOK44244426mirror.0trial15120.3.0-5336-g802bf111falseOKtest959deb13-de1a-48d9-a341-1f95416f8379up+replayingreplaying, {"bytes_per_second":10420381.87,"entries_behind_primary":3709,"entries_per_second":652.0,"non_primary_position":{"entry_tid":25522,"object_number":246,"tag_tid":17},"primary_position":{"entry_tid":1,"object_number":249,"tag_tid":18},"seconds_until_synced":5}42064262mirror.1trial1512026-02-19 20:37:17cluster2a054b997-3cfb-48e6-978a-70d6a85e56deup+stoppedlocal image is primary2026-02-19 20:37:18' 2026-02-19T20:37:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update state 2026-02-19T20:37:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-19T20:37:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:37:17' 2026-02-19T20:37:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-19T20:37:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=up+replaying 2026-02-19T20:37:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:37:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep up+replaying 2026-02-19T20:37:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:37:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:37:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap7 2026-02-19T20:37:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap7 2026-02-19T20:37:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:37:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:37:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:37:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap7 2026-02-19T20:37:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:37:54.703 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:37:54.695+0000 7f5ce3652640 -1 reset not still connected to 0x561025827110 2026-02-19T20:37:54.704 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:37:54.695+0000 7f5ce3652640 -1 reset not still connected to 0x561036b09ee0 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:38:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:38:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:38:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:38:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:38:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:38:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:38:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:38:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:38:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:38:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:38:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:38:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:38:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:38:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:19.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:38:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:38:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:19.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:19.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:38:19.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:38:19.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:38:19.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:38:19.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:38:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:38:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:38:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:38:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:38:19.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:38:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:38:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:38:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:38:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:38:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:38:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:38:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:20.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:20.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 68 2026-02-19T20:38:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=273, tag_tid=20, entry_tid=26449], [object_number=272, tag_tid=20, entry_tid=26448], [object_number=275, tag_tid=20, entry_tid=26447], [object_number=274, tag_tid=20, entry_tid=26446]]], state=connected] 2026-02-19T20:38:20.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:20.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:20.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 68 2026-02-19T20:38:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=273, tag_tid=20, entry_tid=26449], [object_number=272, tag_tid=20, entry_tid=26448], [object_number=275, tag_tid=20, entry_tid=26447], [object_number=274, tag_tid=20, entry_tid=26446]]], state=connected] 2026-02-19T20:38:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=273, tag_tid=20, entry_tid=26449' 2026-02-19T20:38:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=273, tag_tid=20, entry_tid=26449' 2026-02-19T20:38:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=273, tag_tid=20, entry_tid=26449' '!=' '' 2026-02-19T20:38:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=273, tag_tid=20, entry_tid=26449' = '' 2026-02-19T20:38:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=273, tag_tid=20, entry_tid=26449' 2026-02-19T20:38:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:20.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=273, tag_tid=20, entry_tid=26449' 2026-02-19T20:38:20.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:20.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=273, tag_tid=20, entry_tid=26449' 2026-02-19T20:38:20.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:20.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:20.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26449 2026-02-19T20:38:20.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 26449 2026-02-19T20:38:20.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:20.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:20.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:20.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:20.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:20.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:20.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:20.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:20.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:20.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:20.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:20.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:20.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:20.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=26630], [object_number=277, tag_tid=20, entry_tid=26629], [object_number=276, tag_tid=20, entry_tid=26628], [object_number=279, tag_tid=20, entry_tid=26627]]], state=connected] 2026-02-19T20:38:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:20.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:20.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=26630], [object_number=277, tag_tid=20, entry_tid=26629], [object_number=276, tag_tid=20, entry_tid=26628], [object_number=279, tag_tid=20, entry_tid=26627]]], state=connected] 2026-02-19T20:38:20.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=278, tag_tid=20, entry_tid=26630' 2026-02-19T20:38:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=278, tag_tid=20, entry_tid=26630' 2026-02-19T20:38:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=26630' '!=' 'object_number=273, tag_tid=20, entry_tid=26449' 2026-02-19T20:38:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=26630' = 'object_number=273, tag_tid=20, entry_tid=26449' 2026-02-19T20:38:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=278, tag_tid=20, entry_tid=26630' 2026-02-19T20:38:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:20.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=26630' 2026-02-19T20:38:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:20.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:20.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=26630' 2026-02-19T20:38:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26630 2026-02-19T20:38:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 26630 2026-02-19T20:38:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:21.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:21.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:21.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:21.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:21.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:21.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=26940], [object_number=279, tag_tid=20, entry_tid=26939], [object_number=278, tag_tid=20, entry_tid=26938], [object_number=277, tag_tid=20, entry_tid=26937]]], state=connected] 2026-02-19T20:38:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=26940], [object_number=279, tag_tid=20, entry_tid=26939], [object_number=278, tag_tid=20, entry_tid=26938], [object_number=277, tag_tid=20, entry_tid=26937]]], state=connected] 2026-02-19T20:38:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=276, tag_tid=20, entry_tid=26940' 2026-02-19T20:38:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=276, tag_tid=20, entry_tid=26940' 2026-02-19T20:38:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=26940' '!=' 'object_number=278, tag_tid=20, entry_tid=26630' 2026-02-19T20:38:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=26940' = 'object_number=278, tag_tid=20, entry_tid=26630' 2026-02-19T20:38:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=276, tag_tid=20, entry_tid=26940' 2026-02-19T20:38:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:21.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=26940' 2026-02-19T20:38:21.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:21.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:21.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:21.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:21.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:21.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=26940' 2026-02-19T20:38:21.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:21.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26940 2026-02-19T20:38:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 26940 2026-02-19T20:38:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:21.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:21.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:21.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:21.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:21.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:21.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:21.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:21.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:21.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:21.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:21.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:21.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:21.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=27258], [object_number=277, tag_tid=20, entry_tid=27257], [object_number=276, tag_tid=20, entry_tid=27256], [object_number=279, tag_tid=20, entry_tid=27255]]], state=connected] 2026-02-19T20:38:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=27258], [object_number=277, tag_tid=20, entry_tid=27257], [object_number=276, tag_tid=20, entry_tid=27256], [object_number=279, tag_tid=20, entry_tid=27255]]], state=connected] 2026-02-19T20:38:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=278, tag_tid=20, entry_tid=27258' 2026-02-19T20:38:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=278, tag_tid=20, entry_tid=27258' 2026-02-19T20:38:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=27258' '!=' 'object_number=276, tag_tid=20, entry_tid=26940' 2026-02-19T20:38:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=27258' = 'object_number=276, tag_tid=20, entry_tid=26940' 2026-02-19T20:38:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=278, tag_tid=20, entry_tid=27258' 2026-02-19T20:38:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:21.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=27258' 2026-02-19T20:38:21.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:21.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=27258' 2026-02-19T20:38:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27258 2026-02-19T20:38:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 27258 2026-02-19T20:38:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:21.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:21.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:21.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:21.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=27552], [object_number=279, tag_tid=20, entry_tid=27551], [object_number=278, tag_tid=20, entry_tid=27550], [object_number=277, tag_tid=20, entry_tid=27549]]], state=connected] 2026-02-19T20:38:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:22.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:22.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:22.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:22.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=27552], [object_number=279, tag_tid=20, entry_tid=27551], [object_number=278, tag_tid=20, entry_tid=27550], [object_number=277, tag_tid=20, entry_tid=27549]]], state=connected] 2026-02-19T20:38:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=276, tag_tid=20, entry_tid=27552' 2026-02-19T20:38:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=276, tag_tid=20, entry_tid=27552' 2026-02-19T20:38:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=27552' '!=' 'object_number=278, tag_tid=20, entry_tid=27258' 2026-02-19T20:38:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=27552' = 'object_number=278, tag_tid=20, entry_tid=27258' 2026-02-19T20:38:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=276, tag_tid=20, entry_tid=27552' 2026-02-19T20:38:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=27552' 2026-02-19T20:38:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=27552' 2026-02-19T20:38:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27552 2026-02-19T20:38:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 27552 2026-02-19T20:38:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:22.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:22.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:22.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:22.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:22.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:22.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=27836], [object_number=279, tag_tid=20, entry_tid=27835], [object_number=278, tag_tid=20, entry_tid=27834], [object_number=277, tag_tid=20, entry_tid=27833]]], state=connected] 2026-02-19T20:38:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=27836], [object_number=279, tag_tid=20, entry_tid=27835], [object_number=278, tag_tid=20, entry_tid=27834], [object_number=277, tag_tid=20, entry_tid=27833]]], state=connected] 2026-02-19T20:38:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=276, tag_tid=20, entry_tid=27836' 2026-02-19T20:38:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=276, tag_tid=20, entry_tid=27836' 2026-02-19T20:38:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=27836' '!=' 'object_number=276, tag_tid=20, entry_tid=27552' 2026-02-19T20:38:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=27836' = 'object_number=276, tag_tid=20, entry_tid=27552' 2026-02-19T20:38:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=276, tag_tid=20, entry_tid=27836' 2026-02-19T20:38:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=27836' 2026-02-19T20:38:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=27836' 2026-02-19T20:38:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:22.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27836 2026-02-19T20:38:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 27836 2026-02-19T20:38:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:22.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:22.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=28086], [object_number=277, tag_tid=20, entry_tid=28085], [object_number=276, tag_tid=20, entry_tid=28084], [object_number=279, tag_tid=20, entry_tid=28083]]], state=connected] 2026-02-19T20:38:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:22.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:22.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:22.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:22.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=28086], [object_number=277, tag_tid=20, entry_tid=28085], [object_number=276, tag_tid=20, entry_tid=28084], [object_number=279, tag_tid=20, entry_tid=28083]]], state=connected] 2026-02-19T20:38:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=278, tag_tid=20, entry_tid=28086' 2026-02-19T20:38:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=278, tag_tid=20, entry_tid=28086' 2026-02-19T20:38:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=28086' '!=' 'object_number=276, tag_tid=20, entry_tid=27836' 2026-02-19T20:38:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=28086' = 'object_number=276, tag_tid=20, entry_tid=27836' 2026-02-19T20:38:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=278, tag_tid=20, entry_tid=28086' 2026-02-19T20:38:22.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:22.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:22.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:22.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=28086' 2026-02-19T20:38:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=28086' 2026-02-19T20:38:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28086 2026-02-19T20:38:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 28086 2026-02-19T20:38:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:23.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:23.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:23.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:23.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:23.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:23.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:23.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=28372], [object_number=279, tag_tid=20, entry_tid=28371], [object_number=278, tag_tid=20, entry_tid=28370], [object_number=277, tag_tid=20, entry_tid=28369]]], state=connected] 2026-02-19T20:38:23.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:23.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:23.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:23.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:23.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=28372], [object_number=279, tag_tid=20, entry_tid=28371], [object_number=278, tag_tid=20, entry_tid=28370], [object_number=277, tag_tid=20, entry_tid=28369]]], state=connected] 2026-02-19T20:38:23.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=276, tag_tid=20, entry_tid=28372' 2026-02-19T20:38:23.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=276, tag_tid=20, entry_tid=28372' 2026-02-19T20:38:23.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=28372' '!=' 'object_number=278, tag_tid=20, entry_tid=28086' 2026-02-19T20:38:23.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:23.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=28372' = 'object_number=278, tag_tid=20, entry_tid=28086' 2026-02-19T20:38:23.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=276, tag_tid=20, entry_tid=28372' 2026-02-19T20:38:23.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:23.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:23.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=28372' 2026-02-19T20:38:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=28372' 2026-02-19T20:38:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28372 2026-02-19T20:38:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 28372 2026-02-19T20:38:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:23.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:23.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:23.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:23.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:23.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:23.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:23.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:23.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:23.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:23.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=28640], [object_number=279, tag_tid=20, entry_tid=28639], [object_number=278, tag_tid=20, entry_tid=28638], [object_number=277, tag_tid=20, entry_tid=28637]]], state=connected] 2026-02-19T20:38:23.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=28640], [object_number=279, tag_tid=20, entry_tid=28639], [object_number=278, tag_tid=20, entry_tid=28638], [object_number=277, tag_tid=20, entry_tid=28637]]], state=connected] 2026-02-19T20:38:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=276, tag_tid=20, entry_tid=28640' 2026-02-19T20:38:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=276, tag_tid=20, entry_tid=28640' 2026-02-19T20:38:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=28640' '!=' 'object_number=276, tag_tid=20, entry_tid=28372' 2026-02-19T20:38:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=28640' = 'object_number=276, tag_tid=20, entry_tid=28372' 2026-02-19T20:38:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=276, tag_tid=20, entry_tid=28640' 2026-02-19T20:38:23.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:23.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=28640' 2026-02-19T20:38:23.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=28640' 2026-02-19T20:38:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28640 2026-02-19T20:38:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 28640 2026-02-19T20:38:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:24.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:24.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:24.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:24.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:24.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:24.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:24.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:24.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:24.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:24.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:24.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:24.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:24.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:24.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:24.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=279, tag_tid=20, entry_tid=28887], [object_number=278, tag_tid=20, entry_tid=28886], [object_number=277, tag_tid=20, entry_tid=28885], [object_number=276, tag_tid=20, entry_tid=28884]]], state=connected] 2026-02-19T20:38:24.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=279, tag_tid=20, entry_tid=28887], [object_number=278, tag_tid=20, entry_tid=28886], [object_number=277, tag_tid=20, entry_tid=28885], [object_number=276, tag_tid=20, entry_tid=28884]]], state=connected] 2026-02-19T20:38:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=279, tag_tid=20, entry_tid=28887' 2026-02-19T20:38:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=279, tag_tid=20, entry_tid=28887' 2026-02-19T20:38:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=279, tag_tid=20, entry_tid=28887' '!=' 'object_number=276, tag_tid=20, entry_tid=28640' 2026-02-19T20:38:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=279, tag_tid=20, entry_tid=28887' = 'object_number=276, tag_tid=20, entry_tid=28640' 2026-02-19T20:38:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=279, tag_tid=20, entry_tid=28887' 2026-02-19T20:38:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:24.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=279, tag_tid=20, entry_tid=28887' 2026-02-19T20:38:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=279, tag_tid=20, entry_tid=28887' 2026-02-19T20:38:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28887 2026-02-19T20:38:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 28887 2026-02-19T20:38:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:24.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:24.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:24.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:24.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:24.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:24.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:24.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:24.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:24.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:24.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:24.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:24.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:24.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:24.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:24.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:24.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:24.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:24.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:24.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:24.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=279, tag_tid=20, entry_tid=29151], [object_number=278, tag_tid=20, entry_tid=29150], [object_number=277, tag_tid=20, entry_tid=29149], [object_number=276, tag_tid=20, entry_tid=29148]]], state=connected] 2026-02-19T20:38:24.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:24.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=279, tag_tid=20, entry_tid=29151], [object_number=278, tag_tid=20, entry_tid=29150], [object_number=277, tag_tid=20, entry_tid=29149], [object_number=276, tag_tid=20, entry_tid=29148]]], state=connected] 2026-02-19T20:38:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=279, tag_tid=20, entry_tid=29151' 2026-02-19T20:38:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=279, tag_tid=20, entry_tid=29151' 2026-02-19T20:38:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=279, tag_tid=20, entry_tid=29151' '!=' 'object_number=279, tag_tid=20, entry_tid=28887' 2026-02-19T20:38:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=279, tag_tid=20, entry_tid=29151' = 'object_number=279, tag_tid=20, entry_tid=28887' 2026-02-19T20:38:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=279, tag_tid=20, entry_tid=29151' 2026-02-19T20:38:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:24.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=279, tag_tid=20, entry_tid=29151' 2026-02-19T20:38:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:24.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=279, tag_tid=20, entry_tid=29151' 2026-02-19T20:38:24.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:24.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:24.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29151 2026-02-19T20:38:24.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 29151 2026-02-19T20:38:24.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:24.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:24.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:24.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:24.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:24.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:24.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:24.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:24.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:24.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:24.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:24.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:24.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:24.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:24.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=29436], [object_number=279, tag_tid=20, entry_tid=29435], [object_number=278, tag_tid=20, entry_tid=29434], [object_number=277, tag_tid=20, entry_tid=29433]]], state=connected] 2026-02-19T20:38:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:25.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:25.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:25.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:25.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=276, tag_tid=20, entry_tid=29436], [object_number=279, tag_tid=20, entry_tid=29435], [object_number=278, tag_tid=20, entry_tid=29434], [object_number=277, tag_tid=20, entry_tid=29433]]], state=connected] 2026-02-19T20:38:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=276, tag_tid=20, entry_tid=29436' 2026-02-19T20:38:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=276, tag_tid=20, entry_tid=29436' 2026-02-19T20:38:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=29436' '!=' 'object_number=279, tag_tid=20, entry_tid=29151' 2026-02-19T20:38:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=276, tag_tid=20, entry_tid=29436' = 'object_number=279, tag_tid=20, entry_tid=29151' 2026-02-19T20:38:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=276, tag_tid=20, entry_tid=29436' 2026-02-19T20:38:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=29436' 2026-02-19T20:38:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=276, tag_tid=20, entry_tid=29436' 2026-02-19T20:38:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:25.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29436 2026-02-19T20:38:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 29436 2026-02-19T20:38:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:25.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=279, tag_tid=20, entry_tid=29751], [object_number=278, tag_tid=20, entry_tid=29750], [object_number=277, tag_tid=20, entry_tid=29749], [object_number=276, tag_tid=20, entry_tid=29748]]], state=connected] 2026-02-19T20:38:25.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:25.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:25.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:25.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:25.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:25.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:25.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:25.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:25.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:25.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:25.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:25.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:25.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=279, tag_tid=20, entry_tid=29751], [object_number=278, tag_tid=20, entry_tid=29750], [object_number=277, tag_tid=20, entry_tid=29749], [object_number=276, tag_tid=20, entry_tid=29748]]], state=connected] 2026-02-19T20:38:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=279, tag_tid=20, entry_tid=29751' 2026-02-19T20:38:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=279, tag_tid=20, entry_tid=29751' 2026-02-19T20:38:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=279, tag_tid=20, entry_tid=29751' '!=' 'object_number=276, tag_tid=20, entry_tid=29436' 2026-02-19T20:38:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=279, tag_tid=20, entry_tid=29751' = 'object_number=276, tag_tid=20, entry_tid=29436' 2026-02-19T20:38:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=279, tag_tid=20, entry_tid=29751' 2026-02-19T20:38:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:25.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:25.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=279, tag_tid=20, entry_tid=29751' 2026-02-19T20:38:25.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:25.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:25.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=279, tag_tid=20, entry_tid=29751' 2026-02-19T20:38:25.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:25.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29751 2026-02-19T20:38:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 29751 2026-02-19T20:38:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:25.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:25.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:25.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:25.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:25.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:25.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:25.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:25.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:25.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:25.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:25.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:25.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:25.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:25.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:25.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:25.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=30034], [object_number=277, tag_tid=20, entry_tid=30033], [object_number=276, tag_tid=20, entry_tid=30032], [object_number=279, tag_tid=20, entry_tid=30031]]], state=connected] 2026-02-19T20:38:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=30034], [object_number=277, tag_tid=20, entry_tid=30033], [object_number=276, tag_tid=20, entry_tid=30032], [object_number=279, tag_tid=20, entry_tid=30031]]], state=connected] 2026-02-19T20:38:25.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:25.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=278, tag_tid=20, entry_tid=30034' 2026-02-19T20:38:25.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=278, tag_tid=20, entry_tid=30034' 2026-02-19T20:38:25.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=30034' '!=' 'object_number=279, tag_tid=20, entry_tid=29751' 2026-02-19T20:38:25.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:25.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=30034' = 'object_number=279, tag_tid=20, entry_tid=29751' 2026-02-19T20:38:25.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=278, tag_tid=20, entry_tid=30034' 2026-02-19T20:38:25.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:25.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=30034' 2026-02-19T20:38:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=30034' 2026-02-19T20:38:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30034 2026-02-19T20:38:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 30034 2026-02-19T20:38:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:26.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:26.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:26.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:26.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:26.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:26.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=30330], [object_number=277, tag_tid=20, entry_tid=30329], [object_number=276, tag_tid=20, entry_tid=30328], [object_number=279, tag_tid=20, entry_tid=30327]]], state=connected] 2026-02-19T20:38:26.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:26.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:26.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:26.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:26.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:26.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:26.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:26.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:26.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:26.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:26.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:26.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:26.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:26.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=30330], [object_number=277, tag_tid=20, entry_tid=30329], [object_number=276, tag_tid=20, entry_tid=30328], [object_number=279, tag_tid=20, entry_tid=30327]]], state=connected] 2026-02-19T20:38:26.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=278, tag_tid=20, entry_tid=30330' 2026-02-19T20:38:26.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=278, tag_tid=20, entry_tid=30330' 2026-02-19T20:38:26.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=30330' '!=' 'object_number=278, tag_tid=20, entry_tid=30034' 2026-02-19T20:38:26.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:26.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=30330' = 'object_number=278, tag_tid=20, entry_tid=30034' 2026-02-19T20:38:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=278, tag_tid=20, entry_tid=30330' 2026-02-19T20:38:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:26.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=30330' 2026-02-19T20:38:26.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:26.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:26.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:26.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:26.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:26.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=30330' 2026-02-19T20:38:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:26.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30330 2026-02-19T20:38:26.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 30330 2026-02-19T20:38:26.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:26.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:26.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:26.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:26.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:26.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:26.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:26.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:26.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:26.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:26.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:26.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:26.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:26.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=30626], [object_number=277, tag_tid=20, entry_tid=30625], [object_number=276, tag_tid=20, entry_tid=30624], [object_number=279, tag_tid=20, entry_tid=30623]]], state=connected] 2026-02-19T20:38:26.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=278, tag_tid=20, entry_tid=30626], [object_number=277, tag_tid=20, entry_tid=30625], [object_number=276, tag_tid=20, entry_tid=30624], [object_number=279, tag_tid=20, entry_tid=30623]]], state=connected] 2026-02-19T20:38:26.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:26.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=278, tag_tid=20, entry_tid=30626' 2026-02-19T20:38:26.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=278, tag_tid=20, entry_tid=30626' 2026-02-19T20:38:26.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=30626' '!=' 'object_number=278, tag_tid=20, entry_tid=30330' 2026-02-19T20:38:26.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:26.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=278, tag_tid=20, entry_tid=30626' = 'object_number=278, tag_tid=20, entry_tid=30330' 2026-02-19T20:38:26.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=278, tag_tid=20, entry_tid=30626' 2026-02-19T20:38:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=30626' 2026-02-19T20:38:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:26.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:26.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:26.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:26.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:26.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:26.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=278, tag_tid=20, entry_tid=30626' 2026-02-19T20:38:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:26.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30626 2026-02-19T20:38:26.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 30626 2026-02-19T20:38:26.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:26.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:26.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:27.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:27.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:27.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:27.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:27.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:27.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:27.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:27.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:27.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:27.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:27.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:27.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:27.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:27.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:27.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:27.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:27.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:27.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:27.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:27.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=283, tag_tid=20, entry_tid=30847], [object_number=278, tag_tid=20, entry_tid=30846], [object_number=277, tag_tid=20, entry_tid=30845], [object_number=276, tag_tid=20, entry_tid=30844]]], state=connected] 2026-02-19T20:38:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 69 2026-02-19T20:38:27.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:27.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:27.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:27.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=283, tag_tid=20, entry_tid=30847], [object_number=278, tag_tid=20, entry_tid=30846], [object_number=277, tag_tid=20, entry_tid=30845], [object_number=276, tag_tid=20, entry_tid=30844]]], state=connected] 2026-02-19T20:38:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=283, tag_tid=20, entry_tid=30847' 2026-02-19T20:38:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=283, tag_tid=20, entry_tid=30847' 2026-02-19T20:38:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=283, tag_tid=20, entry_tid=30847' '!=' 'object_number=278, tag_tid=20, entry_tid=30626' 2026-02-19T20:38:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=283, tag_tid=20, entry_tid=30847' = 'object_number=278, tag_tid=20, entry_tid=30626' 2026-02-19T20:38:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=283, tag_tid=20, entry_tid=30847' 2026-02-19T20:38:27.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:27.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:27.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:27.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:27.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=283, tag_tid=20, entry_tid=30847' 2026-02-19T20:38:27.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:27.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:27.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:27.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:27.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:27.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=283, tag_tid=20, entry_tid=30847' 2026-02-19T20:38:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:27.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30847 2026-02-19T20:38:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 30847 2026-02-19T20:38:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:27.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:27.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:27.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:27.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:27.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:27.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:27.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:27.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:27.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:27.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:27.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:27.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:27.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:27.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:27.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:27.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:27.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:27.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:27.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:27.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:27.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:27.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:27.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:27.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=280, tag_tid=20, entry_tid=31068], [object_number=283, tag_tid=20, entry_tid=31067], [object_number=282, tag_tid=20, entry_tid=31066], [object_number=281, tag_tid=20, entry_tid=31065]]], state=connected] 2026-02-19T20:38:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:27.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:27.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:27.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:27.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=280, tag_tid=20, entry_tid=31068], [object_number=283, tag_tid=20, entry_tid=31067], [object_number=282, tag_tid=20, entry_tid=31066], [object_number=281, tag_tid=20, entry_tid=31065]]], state=connected] 2026-02-19T20:38:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=280, tag_tid=20, entry_tid=31068' 2026-02-19T20:38:27.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=280, tag_tid=20, entry_tid=31068' 2026-02-19T20:38:27.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=280, tag_tid=20, entry_tid=31068' '!=' 'object_number=283, tag_tid=20, entry_tid=30847' 2026-02-19T20:38:27.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:27.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=280, tag_tid=20, entry_tid=31068' = 'object_number=283, tag_tid=20, entry_tid=30847' 2026-02-19T20:38:27.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=280, tag_tid=20, entry_tid=31068' 2026-02-19T20:38:27.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:27.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:27.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:27.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:27.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=280, tag_tid=20, entry_tid=31068' 2026-02-19T20:38:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:27.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:27.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:27.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:27.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=280, tag_tid=20, entry_tid=31068' 2026-02-19T20:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31068 2026-02-19T20:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 31068 2026-02-19T20:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=280, tag_tid=20, entry_tid=31356], [object_number=283, tag_tid=20, entry_tid=31355], [object_number=282, tag_tid=20, entry_tid=31354], [object_number=281, tag_tid=20, entry_tid=31353]]], state=connected] 2026-02-19T20:38:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:28.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:28.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=280, tag_tid=20, entry_tid=31356], [object_number=283, tag_tid=20, entry_tid=31355], [object_number=282, tag_tid=20, entry_tid=31354], [object_number=281, tag_tid=20, entry_tid=31353]]], state=connected] 2026-02-19T20:38:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=280, tag_tid=20, entry_tid=31356' 2026-02-19T20:38:28.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=280, tag_tid=20, entry_tid=31356' 2026-02-19T20:38:28.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=280, tag_tid=20, entry_tid=31356' '!=' 'object_number=280, tag_tid=20, entry_tid=31068' 2026-02-19T20:38:28.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:28.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=280, tag_tid=20, entry_tid=31356' = 'object_number=280, tag_tid=20, entry_tid=31068' 2026-02-19T20:38:28.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=280, tag_tid=20, entry_tid=31356' 2026-02-19T20:38:28.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:28.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:28.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=280, tag_tid=20, entry_tid=31356' 2026-02-19T20:38:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:28.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=280, tag_tid=20, entry_tid=31356' 2026-02-19T20:38:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31356 2026-02-19T20:38:28.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 31356 2026-02-19T20:38:28.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:28.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:28.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:28.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:28.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:28.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:28.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:28.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:28.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:28.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=281, tag_tid=20, entry_tid=31657], [object_number=280, tag_tid=20, entry_tid=31656], [object_number=283, tag_tid=20, entry_tid=31655], [object_number=282, tag_tid=20, entry_tid=31654]]], state=connected] 2026-02-19T20:38:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=281, tag_tid=20, entry_tid=31657], [object_number=280, tag_tid=20, entry_tid=31656], [object_number=283, tag_tid=20, entry_tid=31655], [object_number=282, tag_tid=20, entry_tid=31654]]], state=connected] 2026-02-19T20:38:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=281, tag_tid=20, entry_tid=31657' 2026-02-19T20:38:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=281, tag_tid=20, entry_tid=31657' 2026-02-19T20:38:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=281, tag_tid=20, entry_tid=31657' '!=' 'object_number=280, tag_tid=20, entry_tid=31356' 2026-02-19T20:38:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=281, tag_tid=20, entry_tid=31657' = 'object_number=280, tag_tid=20, entry_tid=31356' 2026-02-19T20:38:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=281, tag_tid=20, entry_tid=31657' 2026-02-19T20:38:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=281, tag_tid=20, entry_tid=31657' 2026-02-19T20:38:28.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:28.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:28.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:28.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:28.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:28.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=281, tag_tid=20, entry_tid=31657' 2026-02-19T20:38:28.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:28.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31657 2026-02-19T20:38:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 31657 2026-02-19T20:38:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:28.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=283, tag_tid=20, entry_tid=31959], [object_number=282, tag_tid=20, entry_tid=31958], [object_number=281, tag_tid=20, entry_tid=31957], [object_number=280, tag_tid=20, entry_tid=31956]]], state=connected] 2026-02-19T20:38:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:28.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:28.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=283, tag_tid=20, entry_tid=31959], [object_number=282, tag_tid=20, entry_tid=31958], [object_number=281, tag_tid=20, entry_tid=31957], [object_number=280, tag_tid=20, entry_tid=31956]]], state=connected] 2026-02-19T20:38:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=283, tag_tid=20, entry_tid=31959' 2026-02-19T20:38:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=283, tag_tid=20, entry_tid=31959' 2026-02-19T20:38:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=283, tag_tid=20, entry_tid=31959' '!=' 'object_number=281, tag_tid=20, entry_tid=31657' 2026-02-19T20:38:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=283, tag_tid=20, entry_tid=31959' = 'object_number=281, tag_tid=20, entry_tid=31657' 2026-02-19T20:38:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=283, tag_tid=20, entry_tid=31959' 2026-02-19T20:38:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:28.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=283, tag_tid=20, entry_tid=31959' 2026-02-19T20:38:28.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:28.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=283, tag_tid=20, entry_tid=31959' 2026-02-19T20:38:28.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:28.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:28.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31959 2026-02-19T20:38:28.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 31959 2026-02-19T20:38:28.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:28.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:28.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:29.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:29.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:29.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:29.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:29.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:29.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:29.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=283, tag_tid=20, entry_tid=32235], [object_number=282, tag_tid=20, entry_tid=32234], [object_number=281, tag_tid=20, entry_tid=32233], [object_number=280, tag_tid=20, entry_tid=32232]]], state=connected] 2026-02-19T20:38:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:29.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:29.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:29.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:29.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=283, tag_tid=20, entry_tid=32235], [object_number=282, tag_tid=20, entry_tid=32234], [object_number=281, tag_tid=20, entry_tid=32233], [object_number=280, tag_tid=20, entry_tid=32232]]], state=connected] 2026-02-19T20:38:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=283, tag_tid=20, entry_tid=32235' 2026-02-19T20:38:29.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=283, tag_tid=20, entry_tid=32235' 2026-02-19T20:38:29.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=283, tag_tid=20, entry_tid=32235' '!=' 'object_number=283, tag_tid=20, entry_tid=31959' 2026-02-19T20:38:29.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:29.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=283, tag_tid=20, entry_tid=32235' = 'object_number=283, tag_tid=20, entry_tid=31959' 2026-02-19T20:38:29.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=283, tag_tid=20, entry_tid=32235' 2026-02-19T20:38:29.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:29.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:29.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:29.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:29.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=283, tag_tid=20, entry_tid=32235' 2026-02-19T20:38:29.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:29.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:29.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:29.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=283, tag_tid=20, entry_tid=32235' 2026-02-19T20:38:29.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:29.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:29.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32235 2026-02-19T20:38:29.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 32235 2026-02-19T20:38:29.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:29.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:29.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:29.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:29.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:29.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:29.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:29.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:29.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:29.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:29.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:29.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:29.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:29.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:29.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:29.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:29.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:29.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:29.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:29.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:29.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:29.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:29.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:29.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=283, tag_tid=20, entry_tid=32547], [object_number=282, tag_tid=20, entry_tid=32546], [object_number=281, tag_tid=20, entry_tid=32545], [object_number=280, tag_tid=20, entry_tid=32544]]], state=connected] 2026-02-19T20:38:29.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:29.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:29.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:29.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:29.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=283, tag_tid=20, entry_tid=32547], [object_number=282, tag_tid=20, entry_tid=32546], [object_number=281, tag_tid=20, entry_tid=32545], [object_number=280, tag_tid=20, entry_tid=32544]]], state=connected] 2026-02-19T20:38:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:29.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=283, tag_tid=20, entry_tid=32547' 2026-02-19T20:38:29.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=283, tag_tid=20, entry_tid=32547' 2026-02-19T20:38:29.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=283, tag_tid=20, entry_tid=32547' '!=' 'object_number=283, tag_tid=20, entry_tid=32235' 2026-02-19T20:38:29.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:29.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=283, tag_tid=20, entry_tid=32547' = 'object_number=283, tag_tid=20, entry_tid=32235' 2026-02-19T20:38:29.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=283, tag_tid=20, entry_tid=32547' 2026-02-19T20:38:29.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:29.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:29.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:29.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=283, tag_tid=20, entry_tid=32547' 2026-02-19T20:38:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:29.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:29.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:29.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:29.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:29.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=283, tag_tid=20, entry_tid=32547' 2026-02-19T20:38:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:29.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32547 2026-02-19T20:38:29.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 32547 2026-02-19T20:38:29.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:29.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:29.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:30.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=281, tag_tid=20, entry_tid=32853], [object_number=280, tag_tid=20, entry_tid=32852], [object_number=283, tag_tid=20, entry_tid=32851], [object_number=282, tag_tid=20, entry_tid=32850]]], state=connected] 2026-02-19T20:38:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=281, tag_tid=20, entry_tid=32853], [object_number=280, tag_tid=20, entry_tid=32852], [object_number=283, tag_tid=20, entry_tid=32851], [object_number=282, tag_tid=20, entry_tid=32850]]], state=connected] 2026-02-19T20:38:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=281, tag_tid=20, entry_tid=32853' 2026-02-19T20:38:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=281, tag_tid=20, entry_tid=32853' 2026-02-19T20:38:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=281, tag_tid=20, entry_tid=32853' '!=' 'object_number=283, tag_tid=20, entry_tid=32547' 2026-02-19T20:38:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=281, tag_tid=20, entry_tid=32853' = 'object_number=283, tag_tid=20, entry_tid=32547' 2026-02-19T20:38:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=281, tag_tid=20, entry_tid=32853' 2026-02-19T20:38:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:30.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:30.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=281, tag_tid=20, entry_tid=32853' 2026-02-19T20:38:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=281, tag_tid=20, entry_tid=32853' 2026-02-19T20:38:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32853 2026-02-19T20:38:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 32853 2026-02-19T20:38:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:30.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:30.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:30.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:30.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:30.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:30.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:30.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:30.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:30.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:30.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:30.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:30.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=282, tag_tid=20, entry_tid=33170], [object_number=281, tag_tid=20, entry_tid=33169], [object_number=280, tag_tid=20, entry_tid=33168], [object_number=283, tag_tid=20, entry_tid=33167]]], state=connected] 2026-02-19T20:38:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:30.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:30.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:30.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:30.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:30.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:30.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:30.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:30.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:30.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=282, tag_tid=20, entry_tid=33170], [object_number=281, tag_tid=20, entry_tid=33169], [object_number=280, tag_tid=20, entry_tid=33168], [object_number=283, tag_tid=20, entry_tid=33167]]], state=connected] 2026-02-19T20:38:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=282, tag_tid=20, entry_tid=33170' 2026-02-19T20:38:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=282, tag_tid=20, entry_tid=33170' 2026-02-19T20:38:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=282, tag_tid=20, entry_tid=33170' '!=' 'object_number=281, tag_tid=20, entry_tid=32853' 2026-02-19T20:38:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=282, tag_tid=20, entry_tid=33170' = 'object_number=281, tag_tid=20, entry_tid=32853' 2026-02-19T20:38:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=282, tag_tid=20, entry_tid=33170' 2026-02-19T20:38:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=282, tag_tid=20, entry_tid=33170' 2026-02-19T20:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:30.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:30.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:30.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:30.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=282, tag_tid=20, entry_tid=33170' 2026-02-19T20:38:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33170 2026-02-19T20:38:30.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 33170 2026-02-19T20:38:30.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:30.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:30.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:30.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:30.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:30.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:30.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:30.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:30.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:30.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:31.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:31.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=282, tag_tid=20, entry_tid=33470], [object_number=281, tag_tid=20, entry_tid=33469], [object_number=280, tag_tid=20, entry_tid=33468], [object_number=283, tag_tid=20, entry_tid=33467]]], state=connected] 2026-02-19T20:38:31.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=282, tag_tid=20, entry_tid=33470], [object_number=281, tag_tid=20, entry_tid=33469], [object_number=280, tag_tid=20, entry_tid=33468], [object_number=283, tag_tid=20, entry_tid=33467]]], state=connected] 2026-02-19T20:38:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=282, tag_tid=20, entry_tid=33470' 2026-02-19T20:38:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=282, tag_tid=20, entry_tid=33470' 2026-02-19T20:38:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=282, tag_tid=20, entry_tid=33470' '!=' 'object_number=282, tag_tid=20, entry_tid=33170' 2026-02-19T20:38:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=282, tag_tid=20, entry_tid=33470' = 'object_number=282, tag_tid=20, entry_tid=33170' 2026-02-19T20:38:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=282, tag_tid=20, entry_tid=33470' 2026-02-19T20:38:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:31.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:31.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=282, tag_tid=20, entry_tid=33470' 2026-02-19T20:38:31.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:31.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:31.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:31.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:31.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:31.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:31.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=282, tag_tid=20, entry_tid=33470' 2026-02-19T20:38:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:31.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33470 2026-02-19T20:38:31.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 33470 2026-02-19T20:38:31.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:31.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:31.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:31.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:31.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:31.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:31.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:31.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:31.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:31.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:31.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:31.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:31.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:31.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:31.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=281, tag_tid=20, entry_tid=33781], [object_number=280, tag_tid=20, entry_tid=33780], [object_number=283, tag_tid=20, entry_tid=33779], [object_number=282, tag_tid=20, entry_tid=33778]]], state=connected] 2026-02-19T20:38:31.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:31.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:31.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:31.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:31.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:31.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:31.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:31.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:31.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=281, tag_tid=20, entry_tid=33781], [object_number=280, tag_tid=20, entry_tid=33780], [object_number=283, tag_tid=20, entry_tid=33779], [object_number=282, tag_tid=20, entry_tid=33778]]], state=connected] 2026-02-19T20:38:31.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=281, tag_tid=20, entry_tid=33781' 2026-02-19T20:38:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=281, tag_tid=20, entry_tid=33781' 2026-02-19T20:38:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=281, tag_tid=20, entry_tid=33781' '!=' 'object_number=282, tag_tid=20, entry_tid=33470' 2026-02-19T20:38:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=281, tag_tid=20, entry_tid=33781' = 'object_number=282, tag_tid=20, entry_tid=33470' 2026-02-19T20:38:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=281, tag_tid=20, entry_tid=33781' 2026-02-19T20:38:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:31.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:31.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=281, tag_tid=20, entry_tid=33781' 2026-02-19T20:38:31.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:31.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:31.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:31.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:31.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:31.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:31.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:31.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=281, tag_tid=20, entry_tid=33781' 2026-02-19T20:38:31.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:31.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33781 2026-02-19T20:38:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 33781 2026-02-19T20:38:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:31.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:31.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:31.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:31.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:31.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:31.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:31.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:31.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:31.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:31.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:31.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:31.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:31.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:31.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:31.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:31.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:31.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=280, tag_tid=20, entry_tid=34060], [object_number=283, tag_tid=20, entry_tid=34059], [object_number=282, tag_tid=20, entry_tid=34058], [object_number=281, tag_tid=20, entry_tid=34057]]], state=connected] 2026-02-19T20:38:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:31.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:31.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:31.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:31.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:31.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=280, tag_tid=20, entry_tid=34060], [object_number=283, tag_tid=20, entry_tid=34059], [object_number=282, tag_tid=20, entry_tid=34058], [object_number=281, tag_tid=20, entry_tid=34057]]], state=connected] 2026-02-19T20:38:31.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:31.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=280, tag_tid=20, entry_tid=34060' 2026-02-19T20:38:31.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=280, tag_tid=20, entry_tid=34060' 2026-02-19T20:38:31.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=280, tag_tid=20, entry_tid=34060' '!=' 'object_number=281, tag_tid=20, entry_tid=33781' 2026-02-19T20:38:31.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:31.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=280, tag_tid=20, entry_tid=34060' = 'object_number=281, tag_tid=20, entry_tid=33781' 2026-02-19T20:38:31.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=280, tag_tid=20, entry_tid=34060' 2026-02-19T20:38:31.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:31.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:31.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=280, tag_tid=20, entry_tid=34060' 2026-02-19T20:38:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:31.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:32.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:32.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:32.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=280, tag_tid=20, entry_tid=34060' 2026-02-19T20:38:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34060 2026-02-19T20:38:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 34060 2026-02-19T20:38:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:32.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:32.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:32.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:32.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:32.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:32.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:32.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:32.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:32.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:32.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:32.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:32.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:32.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:32.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:32.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:32.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:32.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:32.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:32.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:32.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=281, tag_tid=20, entry_tid=34349], [object_number=280, tag_tid=20, entry_tid=34348], [object_number=283, tag_tid=20, entry_tid=34347], [object_number=282, tag_tid=20, entry_tid=34346]]], state=connected] 2026-02-19T20:38:32.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:32.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:32.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:32.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:32.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:32.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:32.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=281, tag_tid=20, entry_tid=34349], [object_number=280, tag_tid=20, entry_tid=34348], [object_number=283, tag_tid=20, entry_tid=34347], [object_number=282, tag_tid=20, entry_tid=34346]]], state=connected] 2026-02-19T20:38:32.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:32.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=281, tag_tid=20, entry_tid=34349' 2026-02-19T20:38:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=281, tag_tid=20, entry_tid=34349' 2026-02-19T20:38:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=281, tag_tid=20, entry_tid=34349' '!=' 'object_number=280, tag_tid=20, entry_tid=34060' 2026-02-19T20:38:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=281, tag_tid=20, entry_tid=34349' = 'object_number=280, tag_tid=20, entry_tid=34060' 2026-02-19T20:38:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=281, tag_tid=20, entry_tid=34349' 2026-02-19T20:38:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=281, tag_tid=20, entry_tid=34349' 2026-02-19T20:38:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=281, tag_tid=20, entry_tid=34349' 2026-02-19T20:38:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34349 2026-02-19T20:38:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 34349 2026-02-19T20:38:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:32.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:32.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=280, tag_tid=20, entry_tid=34844], [object_number=283, tag_tid=20, entry_tid=34843], [object_number=282, tag_tid=20, entry_tid=34842], [object_number=281, tag_tid=20, entry_tid=34841]]], state=connected] 2026-02-19T20:38:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 70 2026-02-19T20:38:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=280, tag_tid=20, entry_tid=34844], [object_number=283, tag_tid=20, entry_tid=34843], [object_number=282, tag_tid=20, entry_tid=34842], [object_number=281, tag_tid=20, entry_tid=34841]]], state=connected] 2026-02-19T20:38:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=280, tag_tid=20, entry_tid=34844' 2026-02-19T20:38:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=280, tag_tid=20, entry_tid=34844' 2026-02-19T20:38:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=280, tag_tid=20, entry_tid=34844' '!=' 'object_number=281, tag_tid=20, entry_tid=34349' 2026-02-19T20:38:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=280, tag_tid=20, entry_tid=34844' = 'object_number=281, tag_tid=20, entry_tid=34349' 2026-02-19T20:38:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=280, tag_tid=20, entry_tid=34844' 2026-02-19T20:38:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=280, tag_tid=20, entry_tid=34844' 2026-02-19T20:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=280, tag_tid=20, entry_tid=34844' 2026-02-19T20:38:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:33.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34844 2026-02-19T20:38:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 34844 2026-02-19T20:38:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:33.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:33.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:33.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:33.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:33.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:33.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:33.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:33.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:33.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:33.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:33.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:33.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:33.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=35028], [object_number=287, tag_tid=20, entry_tid=35027], [object_number=286, tag_tid=20, entry_tid=35026], [object_number=285, tag_tid=20, entry_tid=35025]]], state=connected] 2026-02-19T20:38:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=35028], [object_number=287, tag_tid=20, entry_tid=35027], [object_number=286, tag_tid=20, entry_tid=35026], [object_number=285, tag_tid=20, entry_tid=35025]]], state=connected] 2026-02-19T20:38:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=284, tag_tid=20, entry_tid=35028' 2026-02-19T20:38:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=284, tag_tid=20, entry_tid=35028' 2026-02-19T20:38:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=35028' '!=' 'object_number=280, tag_tid=20, entry_tid=34844' 2026-02-19T20:38:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=35028' = 'object_number=280, tag_tid=20, entry_tid=34844' 2026-02-19T20:38:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=284, tag_tid=20, entry_tid=35028' 2026-02-19T20:38:33.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:33.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:33.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:33.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=35028' 2026-02-19T20:38:33.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:33.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:33.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:33.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:33.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=35028' 2026-02-19T20:38:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35028 2026-02-19T20:38:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 35028 2026-02-19T20:38:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:33.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:33.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:33.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:33.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:33.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:33.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:33.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:33.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:33.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:33.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:33.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:33.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:33.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:33.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:33.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:33.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:33.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:33.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:33.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:33.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=35328], [object_number=287, tag_tid=20, entry_tid=35327], [object_number=286, tag_tid=20, entry_tid=35326], [object_number=285, tag_tid=20, entry_tid=35325]]], state=connected] 2026-02-19T20:38:33.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:33.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:33.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:33.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:33.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:33.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:33.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:33.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:33.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:33.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:33.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:33.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:33.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=35328], [object_number=287, tag_tid=20, entry_tid=35327], [object_number=286, tag_tid=20, entry_tid=35326], [object_number=285, tag_tid=20, entry_tid=35325]]], state=connected] 2026-02-19T20:38:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=284, tag_tid=20, entry_tid=35328' 2026-02-19T20:38:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=284, tag_tid=20, entry_tid=35328' 2026-02-19T20:38:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=35328' '!=' 'object_number=284, tag_tid=20, entry_tid=35028' 2026-02-19T20:38:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=35328' = 'object_number=284, tag_tid=20, entry_tid=35028' 2026-02-19T20:38:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=284, tag_tid=20, entry_tid=35328' 2026-02-19T20:38:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=35328' 2026-02-19T20:38:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:33.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:33.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:33.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=35328' 2026-02-19T20:38:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35328 2026-02-19T20:38:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 35328 2026-02-19T20:38:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:33.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:34.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:34.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:34.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:34.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:34.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:34.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:34.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:34.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:34.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:34.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:34.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:34.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:34.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:34.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:34.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:34.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:34.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:34.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:34.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:34.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=35628], [object_number=287, tag_tid=20, entry_tid=35627], [object_number=286, tag_tid=20, entry_tid=35626], [object_number=285, tag_tid=20, entry_tid=35625]]], state=connected] 2026-02-19T20:38:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:34.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:34.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:34.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:34.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:34.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:34.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:34.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:34.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:34.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:34.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:34.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:34.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:34.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:34.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=35628], [object_number=287, tag_tid=20, entry_tid=35627], [object_number=286, tag_tid=20, entry_tid=35626], [object_number=285, tag_tid=20, entry_tid=35625]]], state=connected] 2026-02-19T20:38:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=284, tag_tid=20, entry_tid=35628' 2026-02-19T20:38:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=284, tag_tid=20, entry_tid=35628' 2026-02-19T20:38:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=35628' '!=' 'object_number=284, tag_tid=20, entry_tid=35328' 2026-02-19T20:38:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=35628' = 'object_number=284, tag_tid=20, entry_tid=35328' 2026-02-19T20:38:34.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=284, tag_tid=20, entry_tid=35628' 2026-02-19T20:38:34.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:34.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:34.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:34.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:34.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=35628' 2026-02-19T20:38:34.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:34.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=35628' 2026-02-19T20:38:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35628 2026-02-19T20:38:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 35628 2026-02-19T20:38:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:34.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:34.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:34.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:34.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:34.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:34.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:34.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:34.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:34.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:34.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:34.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=285, tag_tid=20, entry_tid=35905], [object_number=284, tag_tid=20, entry_tid=35904], [object_number=287, tag_tid=20, entry_tid=35903], [object_number=286, tag_tid=20, entry_tid=35902]]], state=connected] 2026-02-19T20:38:34.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:34.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:34.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:34.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:34.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=285, tag_tid=20, entry_tid=35905], [object_number=284, tag_tid=20, entry_tid=35904], [object_number=287, tag_tid=20, entry_tid=35903], [object_number=286, tag_tid=20, entry_tid=35902]]], state=connected] 2026-02-19T20:38:34.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:34.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=285, tag_tid=20, entry_tid=35905' 2026-02-19T20:38:34.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=285, tag_tid=20, entry_tid=35905' 2026-02-19T20:38:34.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=285, tag_tid=20, entry_tid=35905' '!=' 'object_number=284, tag_tid=20, entry_tid=35628' 2026-02-19T20:38:34.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:34.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=285, tag_tid=20, entry_tid=35905' = 'object_number=284, tag_tid=20, entry_tid=35628' 2026-02-19T20:38:34.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=285, tag_tid=20, entry_tid=35905' 2026-02-19T20:38:34.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:34.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=285, tag_tid=20, entry_tid=35905' 2026-02-19T20:38:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:34.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:34.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=285, tag_tid=20, entry_tid=35905' 2026-02-19T20:38:34.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:34.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:34.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35905 2026-02-19T20:38:34.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 35905 2026-02-19T20:38:34.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:34.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:34.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:34.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:34.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:34.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:34.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:35.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:35.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:35.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:35.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:35.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:35.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:35.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:35.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:35.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:35.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:35.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:35.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=36164], [object_number=287, tag_tid=20, entry_tid=36163], [object_number=286, tag_tid=20, entry_tid=36162], [object_number=285, tag_tid=20, entry_tid=36161]]], state=connected] 2026-02-19T20:38:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:35.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=36164], [object_number=287, tag_tid=20, entry_tid=36163], [object_number=286, tag_tid=20, entry_tid=36162], [object_number=285, tag_tid=20, entry_tid=36161]]], state=connected] 2026-02-19T20:38:35.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=284, tag_tid=20, entry_tid=36164' 2026-02-19T20:38:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=284, tag_tid=20, entry_tid=36164' 2026-02-19T20:38:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=36164' '!=' 'object_number=285, tag_tid=20, entry_tid=35905' 2026-02-19T20:38:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=36164' = 'object_number=285, tag_tid=20, entry_tid=35905' 2026-02-19T20:38:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=284, tag_tid=20, entry_tid=36164' 2026-02-19T20:38:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:35.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=36164' 2026-02-19T20:38:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:35.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=36164' 2026-02-19T20:38:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36164 2026-02-19T20:38:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 36164 2026-02-19T20:38:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:35.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:35.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:35.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:35.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:35.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:35.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:35.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=285, tag_tid=20, entry_tid=36441], [object_number=284, tag_tid=20, entry_tid=36440], [object_number=287, tag_tid=20, entry_tid=36439], [object_number=286, tag_tid=20, entry_tid=36438]]], state=connected] 2026-02-19T20:38:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=285, tag_tid=20, entry_tid=36441], [object_number=284, tag_tid=20, entry_tid=36440], [object_number=287, tag_tid=20, entry_tid=36439], [object_number=286, tag_tid=20, entry_tid=36438]]], state=connected] 2026-02-19T20:38:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=285, tag_tid=20, entry_tid=36441' 2026-02-19T20:38:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=285, tag_tid=20, entry_tid=36441' 2026-02-19T20:38:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=285, tag_tid=20, entry_tid=36441' '!=' 'object_number=284, tag_tid=20, entry_tid=36164' 2026-02-19T20:38:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=285, tag_tid=20, entry_tid=36441' = 'object_number=284, tag_tid=20, entry_tid=36164' 2026-02-19T20:38:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=285, tag_tid=20, entry_tid=36441' 2026-02-19T20:38:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=285, tag_tid=20, entry_tid=36441' 2026-02-19T20:38:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=285, tag_tid=20, entry_tid=36441' 2026-02-19T20:38:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36441 2026-02-19T20:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 36441 2026-02-19T20:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:35.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:35.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:35.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:35.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:35.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:35.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:35.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:35.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:35.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:35.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:35.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:35.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:35.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:35.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:35.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:35.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:35.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:35.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:35.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:36.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:36.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:36.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:36.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=285, tag_tid=20, entry_tid=36733], [object_number=284, tag_tid=20, entry_tid=36732], [object_number=287, tag_tid=20, entry_tid=36731], [object_number=286, tag_tid=20, entry_tid=36730]]], state=connected] 2026-02-19T20:38:36.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:36.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:36.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:36.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:36.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=285, tag_tid=20, entry_tid=36733], [object_number=284, tag_tid=20, entry_tid=36732], [object_number=287, tag_tid=20, entry_tid=36731], [object_number=286, tag_tid=20, entry_tid=36730]]], state=connected] 2026-02-19T20:38:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=285, tag_tid=20, entry_tid=36733' 2026-02-19T20:38:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=285, tag_tid=20, entry_tid=36733' 2026-02-19T20:38:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=285, tag_tid=20, entry_tid=36733' '!=' 'object_number=285, tag_tid=20, entry_tid=36441' 2026-02-19T20:38:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=285, tag_tid=20, entry_tid=36733' = 'object_number=285, tag_tid=20, entry_tid=36441' 2026-02-19T20:38:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=285, tag_tid=20, entry_tid=36733' 2026-02-19T20:38:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=285, tag_tid=20, entry_tid=36733' 2026-02-19T20:38:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:36.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:36.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:36.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:36.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=285, tag_tid=20, entry_tid=36733' 2026-02-19T20:38:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:36.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36733 2026-02-19T20:38:36.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 36733 2026-02-19T20:38:36.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:36.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:36.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:36.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:36.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:36.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:36.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:36.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:36.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:36.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:36.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:36.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:36.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:36.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:36.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:36.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:36.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:36.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:36.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:36.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:36.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:36.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=286, tag_tid=20, entry_tid=37030], [object_number=285, tag_tid=20, entry_tid=37029], [object_number=284, tag_tid=20, entry_tid=37028], [object_number=287, tag_tid=20, entry_tid=37027]]], state=connected] 2026-02-19T20:38:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:36.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:36.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:36.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:36.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=286, tag_tid=20, entry_tid=37030], [object_number=285, tag_tid=20, entry_tid=37029], [object_number=284, tag_tid=20, entry_tid=37028], [object_number=287, tag_tid=20, entry_tid=37027]]], state=connected] 2026-02-19T20:38:36.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=286, tag_tid=20, entry_tid=37030' 2026-02-19T20:38:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=286, tag_tid=20, entry_tid=37030' 2026-02-19T20:38:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=286, tag_tid=20, entry_tid=37030' '!=' 'object_number=285, tag_tid=20, entry_tid=36733' 2026-02-19T20:38:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=286, tag_tid=20, entry_tid=37030' = 'object_number=285, tag_tid=20, entry_tid=36733' 2026-02-19T20:38:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=286, tag_tid=20, entry_tid=37030' 2026-02-19T20:38:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=286, tag_tid=20, entry_tid=37030' 2026-02-19T20:38:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:36.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:36.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=286, tag_tid=20, entry_tid=37030' 2026-02-19T20:38:36.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:36.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37030 2026-02-19T20:38:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 37030 2026-02-19T20:38:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:36.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:36.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:36.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:36.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:36.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:36.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:36.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:36.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=37340], [object_number=287, tag_tid=20, entry_tid=37339], [object_number=286, tag_tid=20, entry_tid=37338], [object_number=285, tag_tid=20, entry_tid=37337]]], state=connected] 2026-02-19T20:38:37.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:37.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:37.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:37.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:37.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=37340], [object_number=287, tag_tid=20, entry_tid=37339], [object_number=286, tag_tid=20, entry_tid=37338], [object_number=285, tag_tid=20, entry_tid=37337]]], state=connected] 2026-02-19T20:38:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=284, tag_tid=20, entry_tid=37340' 2026-02-19T20:38:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=284, tag_tid=20, entry_tid=37340' 2026-02-19T20:38:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=37340' '!=' 'object_number=286, tag_tid=20, entry_tid=37030' 2026-02-19T20:38:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=37340' = 'object_number=286, tag_tid=20, entry_tid=37030' 2026-02-19T20:38:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=284, tag_tid=20, entry_tid=37340' 2026-02-19T20:38:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=37340' 2026-02-19T20:38:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=37340' 2026-02-19T20:38:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37340 2026-02-19T20:38:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 37340 2026-02-19T20:38:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:37.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:37.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:37.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:37.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:37.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:37.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:37.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:37.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=287, tag_tid=20, entry_tid=37743], [object_number=286, tag_tid=20, entry_tid=37742], [object_number=285, tag_tid=20, entry_tid=37741], [object_number=284, tag_tid=20, entry_tid=37740]]], state=connected] 2026-02-19T20:38:37.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=287, tag_tid=20, entry_tid=37743], [object_number=286, tag_tid=20, entry_tid=37742], [object_number=285, tag_tid=20, entry_tid=37741], [object_number=284, tag_tid=20, entry_tid=37740]]], state=connected] 2026-02-19T20:38:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=287, tag_tid=20, entry_tid=37743' 2026-02-19T20:38:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=287, tag_tid=20, entry_tid=37743' 2026-02-19T20:38:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=287, tag_tid=20, entry_tid=37743' '!=' 'object_number=284, tag_tid=20, entry_tid=37340' 2026-02-19T20:38:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=287, tag_tid=20, entry_tid=37743' = 'object_number=284, tag_tid=20, entry_tid=37340' 2026-02-19T20:38:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=287, tag_tid=20, entry_tid=37743' 2026-02-19T20:38:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=287, tag_tid=20, entry_tid=37743' 2026-02-19T20:38:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:37.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=287, tag_tid=20, entry_tid=37743' 2026-02-19T20:38:37.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:37.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37743 2026-02-19T20:38:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 37743 2026-02-19T20:38:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=286, tag_tid=20, entry_tid=38018], [object_number=285, tag_tid=20, entry_tid=38017], [object_number=284, tag_tid=20, entry_tid=38016], [object_number=287, tag_tid=20, entry_tid=38015]]], state=connected] 2026-02-19T20:38:37.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=286, tag_tid=20, entry_tid=38018], [object_number=285, tag_tid=20, entry_tid=38017], [object_number=284, tag_tid=20, entry_tid=38016], [object_number=287, tag_tid=20, entry_tid=38015]]], state=connected] 2026-02-19T20:38:37.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=286, tag_tid=20, entry_tid=38018' 2026-02-19T20:38:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=286, tag_tid=20, entry_tid=38018' 2026-02-19T20:38:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=286, tag_tid=20, entry_tid=38018' '!=' 'object_number=287, tag_tid=20, entry_tid=37743' 2026-02-19T20:38:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=286, tag_tid=20, entry_tid=38018' = 'object_number=287, tag_tid=20, entry_tid=37743' 2026-02-19T20:38:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=286, tag_tid=20, entry_tid=38018' 2026-02-19T20:38:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=286, tag_tid=20, entry_tid=38018' 2026-02-19T20:38:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=286, tag_tid=20, entry_tid=38018' 2026-02-19T20:38:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38018 2026-02-19T20:38:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 38018 2026-02-19T20:38:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:38.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:38.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:38.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:38.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:38.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:38.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:38.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:38.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:38.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:38.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:38.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=38292], [object_number=287, tag_tid=20, entry_tid=38291], [object_number=286, tag_tid=20, entry_tid=38290], [object_number=285, tag_tid=20, entry_tid=38289]]], state=connected] 2026-02-19T20:38:38.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:38.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=38292], [object_number=287, tag_tid=20, entry_tid=38291], [object_number=286, tag_tid=20, entry_tid=38290], [object_number=285, tag_tid=20, entry_tid=38289]]], state=connected] 2026-02-19T20:38:38.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=284, tag_tid=20, entry_tid=38292' 2026-02-19T20:38:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=284, tag_tid=20, entry_tid=38292' 2026-02-19T20:38:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=38292' '!=' 'object_number=286, tag_tid=20, entry_tid=38018' 2026-02-19T20:38:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=38292' = 'object_number=286, tag_tid=20, entry_tid=38018' 2026-02-19T20:38:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=284, tag_tid=20, entry_tid=38292' 2026-02-19T20:38:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:38.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:38.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=38292' 2026-02-19T20:38:38.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:38.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:38.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:38.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:38.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:38.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:38.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=38292' 2026-02-19T20:38:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38292 2026-02-19T20:38:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 38292 2026-02-19T20:38:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:38.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:38.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:38.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:38.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:38.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:38.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:38.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:38.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:38.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:38.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:38.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=285, tag_tid=20, entry_tid=38593], [object_number=284, tag_tid=20, entry_tid=38592], [object_number=287, tag_tid=20, entry_tid=38591], [object_number=286, tag_tid=20, entry_tid=38590]]], state=connected] 2026-02-19T20:38:38.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:38.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:38.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:38.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:38.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=285, tag_tid=20, entry_tid=38593], [object_number=284, tag_tid=20, entry_tid=38592], [object_number=287, tag_tid=20, entry_tid=38591], [object_number=286, tag_tid=20, entry_tid=38590]]], state=connected] 2026-02-19T20:38:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=285, tag_tid=20, entry_tid=38593' 2026-02-19T20:38:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=285, tag_tid=20, entry_tid=38593' 2026-02-19T20:38:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=285, tag_tid=20, entry_tid=38593' '!=' 'object_number=284, tag_tid=20, entry_tid=38292' 2026-02-19T20:38:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=285, tag_tid=20, entry_tid=38593' = 'object_number=284, tag_tid=20, entry_tid=38292' 2026-02-19T20:38:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=285, tag_tid=20, entry_tid=38593' 2026-02-19T20:38:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=285, tag_tid=20, entry_tid=38593' 2026-02-19T20:38:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=285, tag_tid=20, entry_tid=38593' 2026-02-19T20:38:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:38.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38593 2026-02-19T20:38:38.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 38593 2026-02-19T20:38:38.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:38.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:38.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:39.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:39.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:39.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:39.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:39.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:39.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:39.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:39.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:39.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:39.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=38880], [object_number=287, tag_tid=20, entry_tid=38879], [object_number=286, tag_tid=20, entry_tid=38878], [object_number=285, tag_tid=20, entry_tid=38877]]], state=connected] 2026-02-19T20:38:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:39.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=284, tag_tid=20, entry_tid=38880], [object_number=287, tag_tid=20, entry_tid=38879], [object_number=286, tag_tid=20, entry_tid=38878], [object_number=285, tag_tid=20, entry_tid=38877]]], state=connected] 2026-02-19T20:38:39.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=284, tag_tid=20, entry_tid=38880' 2026-02-19T20:38:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=284, tag_tid=20, entry_tid=38880' 2026-02-19T20:38:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=38880' '!=' 'object_number=285, tag_tid=20, entry_tid=38593' 2026-02-19T20:38:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=284, tag_tid=20, entry_tid=38880' = 'object_number=285, tag_tid=20, entry_tid=38593' 2026-02-19T20:38:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=284, tag_tid=20, entry_tid=38880' 2026-02-19T20:38:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=38880' 2026-02-19T20:38:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=284, tag_tid=20, entry_tid=38880' 2026-02-19T20:38:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38880 2026-02-19T20:38:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 38880 2026-02-19T20:38:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:39.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:39.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:39.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:39.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:39.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:39.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:39.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:39.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:39.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:39.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:39.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:39.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:39.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:39.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:39.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:39.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:39.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:39.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=287, tag_tid=20, entry_tid=39151], [object_number=286, tag_tid=20, entry_tid=39150], [object_number=285, tag_tid=20, entry_tid=39149], [object_number=284, tag_tid=20, entry_tid=39148]]], state=connected] 2026-02-19T20:38:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 71 2026-02-19T20:38:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=287, tag_tid=20, entry_tid=39151], [object_number=286, tag_tid=20, entry_tid=39150], [object_number=285, tag_tid=20, entry_tid=39149], [object_number=284, tag_tid=20, entry_tid=39148]]], state=connected] 2026-02-19T20:38:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=287, tag_tid=20, entry_tid=39151' 2026-02-19T20:38:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=287, tag_tid=20, entry_tid=39151' 2026-02-19T20:38:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=287, tag_tid=20, entry_tid=39151' '!=' 'object_number=284, tag_tid=20, entry_tid=38880' 2026-02-19T20:38:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=287, tag_tid=20, entry_tid=39151' = 'object_number=284, tag_tid=20, entry_tid=38880' 2026-02-19T20:38:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=287, tag_tid=20, entry_tid=39151' 2026-02-19T20:38:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=287, tag_tid=20, entry_tid=39151' 2026-02-19T20:38:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:39.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:39.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=287, tag_tid=20, entry_tid=39151' 2026-02-19T20:38:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39151 2026-02-19T20:38:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 39151 2026-02-19T20:38:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:40.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:40.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:40.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:40.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:40.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=39364], [object_number=291, tag_tid=20, entry_tid=39363], [object_number=290, tag_tid=20, entry_tid=39362], [object_number=289, tag_tid=20, entry_tid=39361]]], state=connected] 2026-02-19T20:38:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=39364], [object_number=291, tag_tid=20, entry_tid=39363], [object_number=290, tag_tid=20, entry_tid=39362], [object_number=289, tag_tid=20, entry_tid=39361]]], state=connected] 2026-02-19T20:38:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=288, tag_tid=20, entry_tid=39364' 2026-02-19T20:38:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=288, tag_tid=20, entry_tid=39364' 2026-02-19T20:38:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=39364' '!=' 'object_number=287, tag_tid=20, entry_tid=39151' 2026-02-19T20:38:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=39364' = 'object_number=287, tag_tid=20, entry_tid=39151' 2026-02-19T20:38:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=288, tag_tid=20, entry_tid=39364' 2026-02-19T20:38:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=39364' 2026-02-19T20:38:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:40.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:40.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:40.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:40.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:40.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=39364' 2026-02-19T20:38:40.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:40.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39364 2026-02-19T20:38:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 39364 2026-02-19T20:38:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:40.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:40.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:40.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:40.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:40.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=39789], [object_number=288, tag_tid=20, entry_tid=39788], [object_number=291, tag_tid=20, entry_tid=39787], [object_number=290, tag_tid=20, entry_tid=39786]]], state=connected] 2026-02-19T20:38:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=39789], [object_number=288, tag_tid=20, entry_tid=39788], [object_number=291, tag_tid=20, entry_tid=39787], [object_number=290, tag_tid=20, entry_tid=39786]]], state=connected] 2026-02-19T20:38:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=289, tag_tid=20, entry_tid=39789' 2026-02-19T20:38:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=289, tag_tid=20, entry_tid=39789' 2026-02-19T20:38:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=39789' '!=' 'object_number=288, tag_tid=20, entry_tid=39364' 2026-02-19T20:38:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=39789' = 'object_number=288, tag_tid=20, entry_tid=39364' 2026-02-19T20:38:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=289, tag_tid=20, entry_tid=39789' 2026-02-19T20:38:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:40.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=39789' 2026-02-19T20:38:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:40.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=39789' 2026-02-19T20:38:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39789 2026-02-19T20:38:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 39789 2026-02-19T20:38:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:41.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:41.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:41.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:41.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:41.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:41.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:41.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=291, tag_tid=20, entry_tid=40035], [object_number=290, tag_tid=20, entry_tid=40034], [object_number=289, tag_tid=20, entry_tid=40033], [object_number=288, tag_tid=20, entry_tid=40032]]], state=connected] 2026-02-19T20:38:41.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=291, tag_tid=20, entry_tid=40035], [object_number=290, tag_tid=20, entry_tid=40034], [object_number=289, tag_tid=20, entry_tid=40033], [object_number=288, tag_tid=20, entry_tid=40032]]], state=connected] 2026-02-19T20:38:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=291, tag_tid=20, entry_tid=40035' 2026-02-19T20:38:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=291, tag_tid=20, entry_tid=40035' 2026-02-19T20:38:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=291, tag_tid=20, entry_tid=40035' '!=' 'object_number=289, tag_tid=20, entry_tid=39789' 2026-02-19T20:38:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=291, tag_tid=20, entry_tid=40035' = 'object_number=289, tag_tid=20, entry_tid=39789' 2026-02-19T20:38:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=291, tag_tid=20, entry_tid=40035' 2026-02-19T20:38:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=291, tag_tid=20, entry_tid=40035' 2026-02-19T20:38:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:41.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=291, tag_tid=20, entry_tid=40035' 2026-02-19T20:38:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40035 2026-02-19T20:38:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 40035 2026-02-19T20:38:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:41.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:41.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:41.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:41.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:41.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:41.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:41.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:41.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:41.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:41.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:41.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:41.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:41.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:41.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=290, tag_tid=20, entry_tid=40290], [object_number=289, tag_tid=20, entry_tid=40289], [object_number=288, tag_tid=20, entry_tid=40288], [object_number=291, tag_tid=20, entry_tid=40287]]], state=connected] 2026-02-19T20:38:41.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:41.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:41.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:41.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:41.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=290, tag_tid=20, entry_tid=40290], [object_number=289, tag_tid=20, entry_tid=40289], [object_number=288, tag_tid=20, entry_tid=40288], [object_number=291, tag_tid=20, entry_tid=40287]]], state=connected] 2026-02-19T20:38:41.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:41.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=290, tag_tid=20, entry_tid=40290' 2026-02-19T20:38:41.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=290, tag_tid=20, entry_tid=40290' 2026-02-19T20:38:41.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=290, tag_tid=20, entry_tid=40290' '!=' 'object_number=291, tag_tid=20, entry_tid=40035' 2026-02-19T20:38:41.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:41.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=290, tag_tid=20, entry_tid=40290' = 'object_number=291, tag_tid=20, entry_tid=40035' 2026-02-19T20:38:41.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=290, tag_tid=20, entry_tid=40290' 2026-02-19T20:38:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=290, tag_tid=20, entry_tid=40290' 2026-02-19T20:38:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:41.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:41.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:41.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:41.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:41.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:41.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=290, tag_tid=20, entry_tid=40290' 2026-02-19T20:38:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:41.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40290 2026-02-19T20:38:41.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 40290 2026-02-19T20:38:41.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:41.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:41.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:41.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:41.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:41.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:41.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:41.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:41.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:41.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:41.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:41.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:41.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:41.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:41.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:41.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:41.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:41.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:41.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:41.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=290, tag_tid=20, entry_tid=40566], [object_number=289, tag_tid=20, entry_tid=40565], [object_number=288, tag_tid=20, entry_tid=40564], [object_number=291, tag_tid=20, entry_tid=40563]]], state=connected] 2026-02-19T20:38:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=290, tag_tid=20, entry_tid=40566], [object_number=289, tag_tid=20, entry_tid=40565], [object_number=288, tag_tid=20, entry_tid=40564], [object_number=291, tag_tid=20, entry_tid=40563]]], state=connected] 2026-02-19T20:38:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=290, tag_tid=20, entry_tid=40566' 2026-02-19T20:38:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=290, tag_tid=20, entry_tid=40566' 2026-02-19T20:38:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=290, tag_tid=20, entry_tid=40566' '!=' 'object_number=290, tag_tid=20, entry_tid=40290' 2026-02-19T20:38:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=290, tag_tid=20, entry_tid=40566' = 'object_number=290, tag_tid=20, entry_tid=40290' 2026-02-19T20:38:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=290, tag_tid=20, entry_tid=40566' 2026-02-19T20:38:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=290, tag_tid=20, entry_tid=40566' 2026-02-19T20:38:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:42.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:42.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:42.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=290, tag_tid=20, entry_tid=40566' 2026-02-19T20:38:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:42.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40566 2026-02-19T20:38:42.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 40566 2026-02-19T20:38:42.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:42.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:42.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:42.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:42.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:42.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:42.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:42.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:42.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:42.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:42.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:42.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:42.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:42.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:42.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:42.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:42.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:42.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=41016], [object_number=291, tag_tid=20, entry_tid=41015], [object_number=290, tag_tid=20, entry_tid=41014], [object_number=289, tag_tid=20, entry_tid=41013]]], state=connected] 2026-02-19T20:38:42.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:42.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:42.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:42.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:42.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:42.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:42.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:42.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:42.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=41016], [object_number=291, tag_tid=20, entry_tid=41015], [object_number=290, tag_tid=20, entry_tid=41014], [object_number=289, tag_tid=20, entry_tid=41013]]], state=connected] 2026-02-19T20:38:42.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:42.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=288, tag_tid=20, entry_tid=41016' 2026-02-19T20:38:42.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=288, tag_tid=20, entry_tid=41016' 2026-02-19T20:38:42.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=41016' '!=' 'object_number=290, tag_tid=20, entry_tid=40566' 2026-02-19T20:38:42.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:42.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=41016' = 'object_number=290, tag_tid=20, entry_tid=40566' 2026-02-19T20:38:42.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=288, tag_tid=20, entry_tid=41016' 2026-02-19T20:38:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:42.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:42.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:42.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=41016' 2026-02-19T20:38:42.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:42.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:42.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:42.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:42.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:42.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=41016' 2026-02-19T20:38:42.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:42.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41016 2026-02-19T20:38:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 41016 2026-02-19T20:38:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:42.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:42.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:42.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:42.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:42.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:42.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:42.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:42.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=41293], [object_number=288, tag_tid=20, entry_tid=41292], [object_number=291, tag_tid=20, entry_tid=41291], [object_number=290, tag_tid=20, entry_tid=41290]]], state=connected] 2026-02-19T20:38:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=41293], [object_number=288, tag_tid=20, entry_tid=41292], [object_number=291, tag_tid=20, entry_tid=41291], [object_number=290, tag_tid=20, entry_tid=41290]]], state=connected] 2026-02-19T20:38:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=289, tag_tid=20, entry_tid=41293' 2026-02-19T20:38:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=289, tag_tid=20, entry_tid=41293' 2026-02-19T20:38:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=41293' '!=' 'object_number=288, tag_tid=20, entry_tid=41016' 2026-02-19T20:38:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=41293' = 'object_number=288, tag_tid=20, entry_tid=41016' 2026-02-19T20:38:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=289, tag_tid=20, entry_tid=41293' 2026-02-19T20:38:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:43.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=41293' 2026-02-19T20:38:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:43.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:43.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=41293' 2026-02-19T20:38:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41293 2026-02-19T20:38:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 41293 2026-02-19T20:38:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:43.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:43.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:43.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:43.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:43.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:43.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:43.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:43.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:43.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:43.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:43.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:43.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:43.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:43.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:43.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:43.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:43.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:43.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:43.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:43.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:43.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:43.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:43.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:43.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:43.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:43.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:43.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:43.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:43.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:43.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:43.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:43.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:43.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:43.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:43.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:43.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:43.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:43.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:43.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:43.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:43.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:43.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=41605], [object_number=288, tag_tid=20, entry_tid=41604], [object_number=291, tag_tid=20, entry_tid=41603], [object_number=290, tag_tid=20, entry_tid=41602]]], state=connected] 2026-02-19T20:38:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=41605], [object_number=288, tag_tid=20, entry_tid=41604], [object_number=291, tag_tid=20, entry_tid=41603], [object_number=290, tag_tid=20, entry_tid=41602]]], state=connected] 2026-02-19T20:38:43.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=289, tag_tid=20, entry_tid=41605' 2026-02-19T20:38:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=289, tag_tid=20, entry_tid=41605' 2026-02-19T20:38:43.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=41605' '!=' 'object_number=289, tag_tid=20, entry_tid=41293' 2026-02-19T20:38:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=41605' = 'object_number=289, tag_tid=20, entry_tid=41293' 2026-02-19T20:38:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=289, tag_tid=20, entry_tid=41605' 2026-02-19T20:38:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=41605' 2026-02-19T20:38:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=41605' 2026-02-19T20:38:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41605 2026-02-19T20:38:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 41605 2026-02-19T20:38:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:43.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:43.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:43.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:43.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:43.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=291, tag_tid=20, entry_tid=41875], [object_number=290, tag_tid=20, entry_tid=41874], [object_number=289, tag_tid=20, entry_tid=41873], [object_number=288, tag_tid=20, entry_tid=41872]]], state=connected] 2026-02-19T20:38:44.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:44.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:44.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:44.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:44.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:44.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:44.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:44.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:44.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:44.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:44.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:44.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:44.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:44.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:44.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=291, tag_tid=20, entry_tid=41875], [object_number=290, tag_tid=20, entry_tid=41874], [object_number=289, tag_tid=20, entry_tid=41873], [object_number=288, tag_tid=20, entry_tid=41872]]], state=connected] 2026-02-19T20:38:44.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=291, tag_tid=20, entry_tid=41875' 2026-02-19T20:38:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=291, tag_tid=20, entry_tid=41875' 2026-02-19T20:38:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=291, tag_tid=20, entry_tid=41875' '!=' 'object_number=289, tag_tid=20, entry_tid=41605' 2026-02-19T20:38:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=291, tag_tid=20, entry_tid=41875' = 'object_number=289, tag_tid=20, entry_tid=41605' 2026-02-19T20:38:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=291, tag_tid=20, entry_tid=41875' 2026-02-19T20:38:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=291, tag_tid=20, entry_tid=41875' 2026-02-19T20:38:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=291, tag_tid=20, entry_tid=41875' 2026-02-19T20:38:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=41875 2026-02-19T20:38:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 41875 2026-02-19T20:38:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:44.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:44.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:44.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:44.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:44.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:44.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:44.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=42148], [object_number=291, tag_tid=20, entry_tid=42147], [object_number=290, tag_tid=20, entry_tid=42146], [object_number=289, tag_tid=20, entry_tid=42145]]], state=connected] 2026-02-19T20:38:44.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:44.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:44.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:44.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=42148], [object_number=291, tag_tid=20, entry_tid=42147], [object_number=290, tag_tid=20, entry_tid=42146], [object_number=289, tag_tid=20, entry_tid=42145]]], state=connected] 2026-02-19T20:38:44.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=288, tag_tid=20, entry_tid=42148' 2026-02-19T20:38:44.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=288, tag_tid=20, entry_tid=42148' 2026-02-19T20:38:44.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=42148' '!=' 'object_number=291, tag_tid=20, entry_tid=41875' 2026-02-19T20:38:44.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:44.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=42148' = 'object_number=291, tag_tid=20, entry_tid=41875' 2026-02-19T20:38:44.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=288, tag_tid=20, entry_tid=42148' 2026-02-19T20:38:44.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:44.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:44.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=42148' 2026-02-19T20:38:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:44.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:44.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:44.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:44.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=42148' 2026-02-19T20:38:44.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:44.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42148 2026-02-19T20:38:44.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 42148 2026-02-19T20:38:44.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:44.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:44.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:44.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:44.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:44.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:44.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:44.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:44.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:44.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:44.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:44.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:44.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:44.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:44.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:44.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:44.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=42417], [object_number=288, tag_tid=20, entry_tid=42416], [object_number=291, tag_tid=20, entry_tid=42415], [object_number=290, tag_tid=20, entry_tid=42414]]], state=connected] 2026-02-19T20:38:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=289, tag_tid=20, entry_tid=42417], [object_number=288, tag_tid=20, entry_tid=42416], [object_number=291, tag_tid=20, entry_tid=42415], [object_number=290, tag_tid=20, entry_tid=42414]]], state=connected] 2026-02-19T20:38:44.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=289, tag_tid=20, entry_tid=42417' 2026-02-19T20:38:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=289, tag_tid=20, entry_tid=42417' 2026-02-19T20:38:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=42417' '!=' 'object_number=288, tag_tid=20, entry_tid=42148' 2026-02-19T20:38:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=289, tag_tid=20, entry_tid=42417' = 'object_number=288, tag_tid=20, entry_tid=42148' 2026-02-19T20:38:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=289, tag_tid=20, entry_tid=42417' 2026-02-19T20:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=42417' 2026-02-19T20:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:44.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:44.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=289, tag_tid=20, entry_tid=42417' 2026-02-19T20:38:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:44.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42417 2026-02-19T20:38:44.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 42417 2026-02-19T20:38:44.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:44.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:44.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:45.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:45.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:45.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:45.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:45.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:45.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:45.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:45.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:45.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:45.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:45.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:45.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:45.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:45.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:45.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:45.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:45.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=42692], [object_number=291, tag_tid=20, entry_tid=42691], [object_number=290, tag_tid=20, entry_tid=42690], [object_number=289, tag_tid=20, entry_tid=42689]]], state=connected] 2026-02-19T20:38:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=42692], [object_number=291, tag_tid=20, entry_tid=42691], [object_number=290, tag_tid=20, entry_tid=42690], [object_number=289, tag_tid=20, entry_tid=42689]]], state=connected] 2026-02-19T20:38:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=288, tag_tid=20, entry_tid=42692' 2026-02-19T20:38:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=288, tag_tid=20, entry_tid=42692' 2026-02-19T20:38:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=42692' '!=' 'object_number=289, tag_tid=20, entry_tid=42417' 2026-02-19T20:38:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=42692' = 'object_number=289, tag_tid=20, entry_tid=42417' 2026-02-19T20:38:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=288, tag_tid=20, entry_tid=42692' 2026-02-19T20:38:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=42692' 2026-02-19T20:38:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=42692' 2026-02-19T20:38:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42692 2026-02-19T20:38:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 42692 2026-02-19T20:38:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:45.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:45.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:45.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:45.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:45.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:45.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:45.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:45.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:45.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:45.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:45.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:45.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:45.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=291, tag_tid=20, entry_tid=42983], [object_number=290, tag_tid=20, entry_tid=42982], [object_number=289, tag_tid=20, entry_tid=42981], [object_number=288, tag_tid=20, entry_tid=42980]]], state=connected] 2026-02-19T20:38:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:45.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:45.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:45.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:45.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:45.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:45.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=291, tag_tid=20, entry_tid=42983], [object_number=290, tag_tid=20, entry_tid=42982], [object_number=289, tag_tid=20, entry_tid=42981], [object_number=288, tag_tid=20, entry_tid=42980]]], state=connected] 2026-02-19T20:38:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=291, tag_tid=20, entry_tid=42983' 2026-02-19T20:38:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=291, tag_tid=20, entry_tid=42983' 2026-02-19T20:38:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=291, tag_tid=20, entry_tid=42983' '!=' 'object_number=288, tag_tid=20, entry_tid=42692' 2026-02-19T20:38:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=291, tag_tid=20, entry_tid=42983' = 'object_number=288, tag_tid=20, entry_tid=42692' 2026-02-19T20:38:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=291, tag_tid=20, entry_tid=42983' 2026-02-19T20:38:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:45.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=291, tag_tid=20, entry_tid=42983' 2026-02-19T20:38:45.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:45.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:45.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:45.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=291, tag_tid=20, entry_tid=42983' 2026-02-19T20:38:45.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:45.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:45.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=42983 2026-02-19T20:38:45.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 42983 2026-02-19T20:38:45.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:45.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:45.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:46.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:46.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:46.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:46.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:46.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:46.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:46.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=43300], [object_number=291, tag_tid=20, entry_tid=43299], [object_number=290, tag_tid=20, entry_tid=43298], [object_number=289, tag_tid=20, entry_tid=43297]]], state=connected] 2026-02-19T20:38:46.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:46.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:46.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:46.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:46.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:46.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:46.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 72 2026-02-19T20:38:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=288, tag_tid=20, entry_tid=43300], [object_number=291, tag_tid=20, entry_tid=43299], [object_number=290, tag_tid=20, entry_tid=43298], [object_number=289, tag_tid=20, entry_tid=43297]]], state=connected] 2026-02-19T20:38:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=288, tag_tid=20, entry_tid=43300' 2026-02-19T20:38:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=288, tag_tid=20, entry_tid=43300' 2026-02-19T20:38:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=43300' '!=' 'object_number=291, tag_tid=20, entry_tid=42983' 2026-02-19T20:38:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=288, tag_tid=20, entry_tid=43300' = 'object_number=291, tag_tid=20, entry_tid=42983' 2026-02-19T20:38:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=288, tag_tid=20, entry_tid=43300' 2026-02-19T20:38:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:46.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=43300' 2026-02-19T20:38:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:46.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=288, tag_tid=20, entry_tid=43300' 2026-02-19T20:38:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43300 2026-02-19T20:38:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 43300 2026-02-19T20:38:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:46.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:46.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:46.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:46.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:46.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:46.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:46.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:46.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:46.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:46.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:46.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:46.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:46.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:46.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:46.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:46.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:46.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:46.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:46.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:46.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:46.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:46.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:46.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:46.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:46.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:46.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:46.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:46.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:46.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:46.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:46.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=293, tag_tid=20, entry_tid=43509], [object_number=292, tag_tid=20, entry_tid=43508], [object_number=295, tag_tid=20, entry_tid=43507], [object_number=294, tag_tid=20, entry_tid=43506]]], state=connected] 2026-02-19T20:38:46.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:46.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:46.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:46.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=293, tag_tid=20, entry_tid=43509], [object_number=292, tag_tid=20, entry_tid=43508], [object_number=295, tag_tid=20, entry_tid=43507], [object_number=294, tag_tid=20, entry_tid=43506]]], state=connected] 2026-02-19T20:38:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=293, tag_tid=20, entry_tid=43509' 2026-02-19T20:38:46.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=293, tag_tid=20, entry_tid=43509' 2026-02-19T20:38:46.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=293, tag_tid=20, entry_tid=43509' '!=' 'object_number=288, tag_tid=20, entry_tid=43300' 2026-02-19T20:38:46.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:46.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=293, tag_tid=20, entry_tid=43509' = 'object_number=288, tag_tid=20, entry_tid=43300' 2026-02-19T20:38:46.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=293, tag_tid=20, entry_tid=43509' 2026-02-19T20:38:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:46.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:46.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:46.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=293, tag_tid=20, entry_tid=43509' 2026-02-19T20:38:46.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:46.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:46.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:46.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:46.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=293, tag_tid=20, entry_tid=43509' 2026-02-19T20:38:46.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:46.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43509 2026-02-19T20:38:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 43509 2026-02-19T20:38:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:46.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:46.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:46.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:46.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:46.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:46.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:46.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:46.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:46.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:46.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:46.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:46.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:46.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:46.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:46.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:46.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:46.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:46.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:46.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:46.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:46.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:46.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:46.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:46.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:46.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=43791], [object_number=294, tag_tid=20, entry_tid=43790], [object_number=293, tag_tid=20, entry_tid=43789], [object_number=292, tag_tid=20, entry_tid=43788]]], state=connected] 2026-02-19T20:38:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:47.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:47.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:47.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:47.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=43791], [object_number=294, tag_tid=20, entry_tid=43790], [object_number=293, tag_tid=20, entry_tid=43789], [object_number=292, tag_tid=20, entry_tid=43788]]], state=connected] 2026-02-19T20:38:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=295, tag_tid=20, entry_tid=43791' 2026-02-19T20:38:47.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=295, tag_tid=20, entry_tid=43791' 2026-02-19T20:38:47.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=43791' '!=' 'object_number=293, tag_tid=20, entry_tid=43509' 2026-02-19T20:38:47.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:47.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=43791' = 'object_number=293, tag_tid=20, entry_tid=43509' 2026-02-19T20:38:47.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=295, tag_tid=20, entry_tid=43791' 2026-02-19T20:38:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=43791' 2026-02-19T20:38:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:47.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:47.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=43791' 2026-02-19T20:38:47.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:47.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:47.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=43791 2026-02-19T20:38:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 43791 2026-02-19T20:38:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:47.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:47.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:47.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:47.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:47.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:47.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:47.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:47.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:47.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:47.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:47.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:47.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:47.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:47.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:47.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:47.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:47.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=44096], [object_number=295, tag_tid=20, entry_tid=44095], [object_number=294, tag_tid=20, entry_tid=44094], [object_number=293, tag_tid=20, entry_tid=44093]]], state=connected] 2026-02-19T20:38:47.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:47.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=44096], [object_number=295, tag_tid=20, entry_tid=44095], [object_number=294, tag_tid=20, entry_tid=44094], [object_number=293, tag_tid=20, entry_tid=44093]]], state=connected] 2026-02-19T20:38:47.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=292, tag_tid=20, entry_tid=44096' 2026-02-19T20:38:47.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=292, tag_tid=20, entry_tid=44096' 2026-02-19T20:38:47.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=44096' '!=' 'object_number=295, tag_tid=20, entry_tid=43791' 2026-02-19T20:38:47.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:47.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=44096' = 'object_number=295, tag_tid=20, entry_tid=43791' 2026-02-19T20:38:47.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=292, tag_tid=20, entry_tid=44096' 2026-02-19T20:38:47.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:47.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:47.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=44096' 2026-02-19T20:38:47.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:47.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:47.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:47.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=44096' 2026-02-19T20:38:47.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:47.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:47.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44096 2026-02-19T20:38:47.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 44096 2026-02-19T20:38:47.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:47.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:47.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:47.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:47.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:47.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:47.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:47.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:47.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:47.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:47.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:47.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:47.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:47.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:47.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:47.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:47.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=44387], [object_number=294, tag_tid=20, entry_tid=44386], [object_number=293, tag_tid=20, entry_tid=44385], [object_number=292, tag_tid=20, entry_tid=44384]]], state=connected] 2026-02-19T20:38:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:47.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=44387], [object_number=294, tag_tid=20, entry_tid=44386], [object_number=293, tag_tid=20, entry_tid=44385], [object_number=292, tag_tid=20, entry_tid=44384]]], state=connected] 2026-02-19T20:38:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=295, tag_tid=20, entry_tid=44387' 2026-02-19T20:38:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=295, tag_tid=20, entry_tid=44387' 2026-02-19T20:38:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=44387' '!=' 'object_number=292, tag_tid=20, entry_tid=44096' 2026-02-19T20:38:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=44387' = 'object_number=292, tag_tid=20, entry_tid=44096' 2026-02-19T20:38:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=295, tag_tid=20, entry_tid=44387' 2026-02-19T20:38:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:48.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=44387' 2026-02-19T20:38:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:48.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:48.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:48.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=44387' 2026-02-19T20:38:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:48.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44387 2026-02-19T20:38:48.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 44387 2026-02-19T20:38:48.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:48.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:48.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:48.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:48.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:48.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:48.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:48.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:48.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:48.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:48.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:48.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:48.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:48.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:48.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:48.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:48.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=44836], [object_number=295, tag_tid=20, entry_tid=44835], [object_number=294, tag_tid=20, entry_tid=44834], [object_number=293, tag_tid=20, entry_tid=44833]]], state=connected] 2026-02-19T20:38:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:48.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:48.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:48.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:48.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:48.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:48.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:48.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:48.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:48.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=44836], [object_number=295, tag_tid=20, entry_tid=44835], [object_number=294, tag_tid=20, entry_tid=44834], [object_number=293, tag_tid=20, entry_tid=44833]]], state=connected] 2026-02-19T20:38:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=292, tag_tid=20, entry_tid=44836' 2026-02-19T20:38:48.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=292, tag_tid=20, entry_tid=44836' 2026-02-19T20:38:48.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=44836' '!=' 'object_number=295, tag_tid=20, entry_tid=44387' 2026-02-19T20:38:48.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:48.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=44836' = 'object_number=295, tag_tid=20, entry_tid=44387' 2026-02-19T20:38:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=292, tag_tid=20, entry_tid=44836' 2026-02-19T20:38:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:48.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:48.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:48.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:48.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=44836' 2026-02-19T20:38:48.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:48.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:48.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:48.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:48.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=44836' 2026-02-19T20:38:48.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:48.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:48.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44836 2026-02-19T20:38:48.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 44836 2026-02-19T20:38:48.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:48.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:48.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:48.675 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.46543853238896105) > weight (0.25) 2026-02-19T20:38:48.675 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4343969799713504) > weight (0.25) 2026-02-19T20:38:48.675 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9212470625115072) > weight (0.25) 2026-02-19T20:38:48.675 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.4345620319449346) > weight (0.25) 2026-02-19T20:38:48.675 INFO:tasks.rbd_mirror_thrash:waiting for 95.45525475632009 secs before thrashing 2026-02-19T20:38:48.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:48.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:48.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:48.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:48.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:48.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:48.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:48.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:48.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:48.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:48.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:48.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:48.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:48.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:48.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=45120], [object_number=295, tag_tid=20, entry_tid=45119], [object_number=294, tag_tid=20, entry_tid=45118], [object_number=293, tag_tid=20, entry_tid=45117]]], state=connected] 2026-02-19T20:38:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=45120], [object_number=295, tag_tid=20, entry_tid=45119], [object_number=294, tag_tid=20, entry_tid=45118], [object_number=293, tag_tid=20, entry_tid=45117]]], state=connected] 2026-02-19T20:38:49.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=292, tag_tid=20, entry_tid=45120' 2026-02-19T20:38:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=292, tag_tid=20, entry_tid=45120' 2026-02-19T20:38:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=45120' '!=' 'object_number=292, tag_tid=20, entry_tid=44836' 2026-02-19T20:38:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=45120' = 'object_number=292, tag_tid=20, entry_tid=44836' 2026-02-19T20:38:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=292, tag_tid=20, entry_tid=45120' 2026-02-19T20:38:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=45120' 2026-02-19T20:38:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:49.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=45120' 2026-02-19T20:38:49.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:49.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45120 2026-02-19T20:38:49.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 45120 2026-02-19T20:38:49.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:49.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:49.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:49.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:49.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:49.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:49.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:49.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:49.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=45404], [object_number=295, tag_tid=20, entry_tid=45403], [object_number=294, tag_tid=20, entry_tid=45402], [object_number=293, tag_tid=20, entry_tid=45401]]], state=connected] 2026-02-19T20:38:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=45404], [object_number=295, tag_tid=20, entry_tid=45403], [object_number=294, tag_tid=20, entry_tid=45402], [object_number=293, tag_tid=20, entry_tid=45401]]], state=connected] 2026-02-19T20:38:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=292, tag_tid=20, entry_tid=45404' 2026-02-19T20:38:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=292, tag_tid=20, entry_tid=45404' 2026-02-19T20:38:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=45404' '!=' 'object_number=292, tag_tid=20, entry_tid=45120' 2026-02-19T20:38:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=45404' = 'object_number=292, tag_tid=20, entry_tid=45120' 2026-02-19T20:38:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=292, tag_tid=20, entry_tid=45404' 2026-02-19T20:38:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=45404' 2026-02-19T20:38:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:49.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=45404' 2026-02-19T20:38:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45404 2026-02-19T20:38:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 45404 2026-02-19T20:38:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:49.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:49.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:49.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:49.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=45667], [object_number=294, tag_tid=20, entry_tid=45666], [object_number=293, tag_tid=20, entry_tid=45665], [object_number=292, tag_tid=20, entry_tid=45664]]], state=connected] 2026-02-19T20:38:49.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:49.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:49.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:49.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:49.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:49.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:49.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:49.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:49.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:49.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:49.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:49.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:49.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=45667], [object_number=294, tag_tid=20, entry_tid=45666], [object_number=293, tag_tid=20, entry_tid=45665], [object_number=292, tag_tid=20, entry_tid=45664]]], state=connected] 2026-02-19T20:38:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=295, tag_tid=20, entry_tid=45667' 2026-02-19T20:38:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=295, tag_tid=20, entry_tid=45667' 2026-02-19T20:38:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=45667' '!=' 'object_number=292, tag_tid=20, entry_tid=45404' 2026-02-19T20:38:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=45667' = 'object_number=292, tag_tid=20, entry_tid=45404' 2026-02-19T20:38:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=295, tag_tid=20, entry_tid=45667' 2026-02-19T20:38:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=45667' 2026-02-19T20:38:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:49.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=45667' 2026-02-19T20:38:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45667 2026-02-19T20:38:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 45667 2026-02-19T20:38:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:50.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:50.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:50.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:50.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:50.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:50.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:50.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:50.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:50.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:50.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=293, tag_tid=20, entry_tid=45933], [object_number=292, tag_tid=20, entry_tid=45932], [object_number=295, tag_tid=20, entry_tid=45931], [object_number=294, tag_tid=20, entry_tid=45930]]], state=connected] 2026-02-19T20:38:50.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:50.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=293, tag_tid=20, entry_tid=45933], [object_number=292, tag_tid=20, entry_tid=45932], [object_number=295, tag_tid=20, entry_tid=45931], [object_number=294, tag_tid=20, entry_tid=45930]]], state=connected] 2026-02-19T20:38:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=293, tag_tid=20, entry_tid=45933' 2026-02-19T20:38:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=293, tag_tid=20, entry_tid=45933' 2026-02-19T20:38:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=293, tag_tid=20, entry_tid=45933' '!=' 'object_number=295, tag_tid=20, entry_tid=45667' 2026-02-19T20:38:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=293, tag_tid=20, entry_tid=45933' = 'object_number=295, tag_tid=20, entry_tid=45667' 2026-02-19T20:38:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=293, tag_tid=20, entry_tid=45933' 2026-02-19T20:38:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=293, tag_tid=20, entry_tid=45933' 2026-02-19T20:38:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=293, tag_tid=20, entry_tid=45933' 2026-02-19T20:38:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45933 2026-02-19T20:38:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 45933 2026-02-19T20:38:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:50.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:50.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:50.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:50.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:50.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:50.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:50.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:50.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:50.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:50.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:50.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:50.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:50.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:50.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:50.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:50.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:50.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:50.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:50.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:50.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:50.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:50.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:50.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:50.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:50.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:50.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:50.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:50.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:50.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=46239], [object_number=294, tag_tid=20, entry_tid=46238], [object_number=293, tag_tid=20, entry_tid=46237], [object_number=292, tag_tid=20, entry_tid=46236]]], state=connected] 2026-02-19T20:38:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=46239], [object_number=294, tag_tid=20, entry_tid=46238], [object_number=293, tag_tid=20, entry_tid=46237], [object_number=292, tag_tid=20, entry_tid=46236]]], state=connected] 2026-02-19T20:38:50.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=295, tag_tid=20, entry_tid=46239' 2026-02-19T20:38:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=295, tag_tid=20, entry_tid=46239' 2026-02-19T20:38:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=46239' '!=' 'object_number=293, tag_tid=20, entry_tid=45933' 2026-02-19T20:38:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=46239' = 'object_number=293, tag_tid=20, entry_tid=45933' 2026-02-19T20:38:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=295, tag_tid=20, entry_tid=46239' 2026-02-19T20:38:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:50.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:50.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=46239' 2026-02-19T20:38:50.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:50.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:50.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:50.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:50.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=46239' 2026-02-19T20:38:50.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:50.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:50.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46239 2026-02-19T20:38:50.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 46239 2026-02-19T20:38:50.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:50.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:50.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:50.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:50.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:50.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:50.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:51.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:51.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=46523], [object_number=294, tag_tid=20, entry_tid=46522], [object_number=293, tag_tid=20, entry_tid=46521], [object_number=292, tag_tid=20, entry_tid=46520]]], state=connected] 2026-02-19T20:38:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:51.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:51.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:51.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:51.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:51.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=46523], [object_number=294, tag_tid=20, entry_tid=46522], [object_number=293, tag_tid=20, entry_tid=46521], [object_number=292, tag_tid=20, entry_tid=46520]]], state=connected] 2026-02-19T20:38:51.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=295, tag_tid=20, entry_tid=46523' 2026-02-19T20:38:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=295, tag_tid=20, entry_tid=46523' 2026-02-19T20:38:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=46523' '!=' 'object_number=295, tag_tid=20, entry_tid=46239' 2026-02-19T20:38:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=46523' = 'object_number=295, tag_tid=20, entry_tid=46239' 2026-02-19T20:38:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=295, tag_tid=20, entry_tid=46523' 2026-02-19T20:38:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:51.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:51.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:51.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=46523' 2026-02-19T20:38:51.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:51.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:51.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:51.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=46523' 2026-02-19T20:38:51.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:51.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46523 2026-02-19T20:38:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 46523 2026-02-19T20:38:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:51.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:51.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:51.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:51.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:51.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:51.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:51.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:51.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:51.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:51.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=46779], [object_number=294, tag_tid=20, entry_tid=46778], [object_number=293, tag_tid=20, entry_tid=46777], [object_number=292, tag_tid=20, entry_tid=46776]]], state=connected] 2026-02-19T20:38:51.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:51.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:51.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=46779], [object_number=294, tag_tid=20, entry_tid=46778], [object_number=293, tag_tid=20, entry_tid=46777], [object_number=292, tag_tid=20, entry_tid=46776]]], state=connected] 2026-02-19T20:38:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=295, tag_tid=20, entry_tid=46779' 2026-02-19T20:38:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=295, tag_tid=20, entry_tid=46779' 2026-02-19T20:38:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=46779' '!=' 'object_number=295, tag_tid=20, entry_tid=46523' 2026-02-19T20:38:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=46779' = 'object_number=295, tag_tid=20, entry_tid=46523' 2026-02-19T20:38:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=295, tag_tid=20, entry_tid=46779' 2026-02-19T20:38:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:51.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:51.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=46779' 2026-02-19T20:38:51.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:51.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:51.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=46779' 2026-02-19T20:38:51.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:51.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46779 2026-02-19T20:38:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 46779 2026-02-19T20:38:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:51.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:51.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:51.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:51.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:51.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:51.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=47095], [object_number=294, tag_tid=20, entry_tid=47094], [object_number=293, tag_tid=20, entry_tid=47093], [object_number=292, tag_tid=20, entry_tid=47092]]], state=connected] 2026-02-19T20:38:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=295, tag_tid=20, entry_tid=47095], [object_number=294, tag_tid=20, entry_tid=47094], [object_number=293, tag_tid=20, entry_tid=47093], [object_number=292, tag_tid=20, entry_tid=47092]]], state=connected] 2026-02-19T20:38:52.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=295, tag_tid=20, entry_tid=47095' 2026-02-19T20:38:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=295, tag_tid=20, entry_tid=47095' 2026-02-19T20:38:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=47095' '!=' 'object_number=295, tag_tid=20, entry_tid=46779' 2026-02-19T20:38:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=295, tag_tid=20, entry_tid=47095' = 'object_number=295, tag_tid=20, entry_tid=46779' 2026-02-19T20:38:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=295, tag_tid=20, entry_tid=47095' 2026-02-19T20:38:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:52.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=47095' 2026-02-19T20:38:52.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:52.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:52.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:52.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:52.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:52.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=295, tag_tid=20, entry_tid=47095' 2026-02-19T20:38:52.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:52.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47095 2026-02-19T20:38:52.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 47095 2026-02-19T20:38:52.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:52.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:52.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:52.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:52.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:52.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:52.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:52.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:52.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:52.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:52.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:52.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:52.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:52.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:52.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=47404], [object_number=295, tag_tid=20, entry_tid=47403], [object_number=294, tag_tid=20, entry_tid=47402], [object_number=293, tag_tid=20, entry_tid=47401]]], state=connected] 2026-02-19T20:38:52.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:52.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:52.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:52.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:52.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:52.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:52.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:52.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:52.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:52.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:52.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:52.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:52.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:52.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:52.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 73 2026-02-19T20:38:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=292, tag_tid=20, entry_tid=47404], [object_number=295, tag_tid=20, entry_tid=47403], [object_number=294, tag_tid=20, entry_tid=47402], [object_number=293, tag_tid=20, entry_tid=47401]]], state=connected] 2026-02-19T20:38:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=292, tag_tid=20, entry_tid=47404' 2026-02-19T20:38:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=292, tag_tid=20, entry_tid=47404' 2026-02-19T20:38:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=47404' '!=' 'object_number=295, tag_tid=20, entry_tid=47095' 2026-02-19T20:38:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=292, tag_tid=20, entry_tid=47404' = 'object_number=295, tag_tid=20, entry_tid=47095' 2026-02-19T20:38:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=292, tag_tid=20, entry_tid=47404' 2026-02-19T20:38:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:52.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=47404' 2026-02-19T20:38:52.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:52.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:52.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:52.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:52.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=292, tag_tid=20, entry_tid=47404' 2026-02-19T20:38:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47404 2026-02-19T20:38:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 47404 2026-02-19T20:38:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:52.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:52.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:52.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:52.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:52.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:52.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:52.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:52.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:52.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:52.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:52.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:52.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:52.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:52.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:52.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:52.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:52.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:52.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:52.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:52.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:52.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:52.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:52.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:52.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:52.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:52.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:52.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:52.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:52.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:52.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:52.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:52.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:52.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:52.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:52.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:52.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=47586], [object_number=297, tag_tid=20, entry_tid=47585], [object_number=296, tag_tid=20, entry_tid=47584], [object_number=299, tag_tid=20, entry_tid=47583]]], state=connected] 2026-02-19T20:38:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:52.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=47586], [object_number=297, tag_tid=20, entry_tid=47585], [object_number=296, tag_tid=20, entry_tid=47584], [object_number=299, tag_tid=20, entry_tid=47583]]], state=connected] 2026-02-19T20:38:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=298, tag_tid=20, entry_tid=47586' 2026-02-19T20:38:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=298, tag_tid=20, entry_tid=47586' 2026-02-19T20:38:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=47586' '!=' 'object_number=292, tag_tid=20, entry_tid=47404' 2026-02-19T20:38:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=47586' = 'object_number=292, tag_tid=20, entry_tid=47404' 2026-02-19T20:38:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=298, tag_tid=20, entry_tid=47586' 2026-02-19T20:38:52.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:52.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:52.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=47586' 2026-02-19T20:38:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:52.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:52.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:52.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:52.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=47586' 2026-02-19T20:38:52.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:52.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:52.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47586 2026-02-19T20:38:52.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 47586 2026-02-19T20:38:52.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:52.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:52.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:53.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:53.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:53.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:53.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:53.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:53.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:53.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:53.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:53.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:53.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:53.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:53.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:53.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:53.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:53.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:53.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=47864], [object_number=299, tag_tid=20, entry_tid=47863], [object_number=298, tag_tid=20, entry_tid=47862], [object_number=297, tag_tid=20, entry_tid=47861]]], state=connected] 2026-02-19T20:38:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:53.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:53.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:53.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:53.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=47864], [object_number=299, tag_tid=20, entry_tid=47863], [object_number=298, tag_tid=20, entry_tid=47862], [object_number=297, tag_tid=20, entry_tid=47861]]], state=connected] 2026-02-19T20:38:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=296, tag_tid=20, entry_tid=47864' 2026-02-19T20:38:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=296, tag_tid=20, entry_tid=47864' 2026-02-19T20:38:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=47864' '!=' 'object_number=298, tag_tid=20, entry_tid=47586' 2026-02-19T20:38:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=47864' = 'object_number=298, tag_tid=20, entry_tid=47586' 2026-02-19T20:38:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=296, tag_tid=20, entry_tid=47864' 2026-02-19T20:38:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=47864' 2026-02-19T20:38:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:53.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:53.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:53.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:53.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:53.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:53.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=47864' 2026-02-19T20:38:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47864 2026-02-19T20:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 47864 2026-02-19T20:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:53.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:53.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:53.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:53.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:53.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:53.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:53.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:53.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:53.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:53.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=48121], [object_number=296, tag_tid=20, entry_tid=48120], [object_number=299, tag_tid=20, entry_tid=48119], [object_number=298, tag_tid=20, entry_tid=48118]]], state=connected] 2026-02-19T20:38:53.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:53.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:53.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:53.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:53.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:53.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:53.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:53.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:53.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=48121], [object_number=296, tag_tid=20, entry_tid=48120], [object_number=299, tag_tid=20, entry_tid=48119], [object_number=298, tag_tid=20, entry_tid=48118]]], state=connected] 2026-02-19T20:38:53.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:53.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=297, tag_tid=20, entry_tid=48121' 2026-02-19T20:38:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=297, tag_tid=20, entry_tid=48121' 2026-02-19T20:38:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=48121' '!=' 'object_number=296, tag_tid=20, entry_tid=47864' 2026-02-19T20:38:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=48121' = 'object_number=296, tag_tid=20, entry_tid=47864' 2026-02-19T20:38:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=297, tag_tid=20, entry_tid=48121' 2026-02-19T20:38:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:53.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:53.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=48121' 2026-02-19T20:38:53.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:53.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:53.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:53.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=48121' 2026-02-19T20:38:53.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:53.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48121 2026-02-19T20:38:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 48121 2026-02-19T20:38:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:54.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:54.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:54.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:54.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:54.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:54.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:54.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:54.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:54.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=48405], [object_number=296, tag_tid=20, entry_tid=48404], [object_number=299, tag_tid=20, entry_tid=48403], [object_number=298, tag_tid=20, entry_tid=48402]]], state=connected] 2026-02-19T20:38:54.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:54.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:54.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:54.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:54.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=48405], [object_number=296, tag_tid=20, entry_tid=48404], [object_number=299, tag_tid=20, entry_tid=48403], [object_number=298, tag_tid=20, entry_tid=48402]]], state=connected] 2026-02-19T20:38:54.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=297, tag_tid=20, entry_tid=48405' 2026-02-19T20:38:54.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=297, tag_tid=20, entry_tid=48405' 2026-02-19T20:38:54.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=48405' '!=' 'object_number=297, tag_tid=20, entry_tid=48121' 2026-02-19T20:38:54.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:54.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=48405' = 'object_number=297, tag_tid=20, entry_tid=48121' 2026-02-19T20:38:54.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=297, tag_tid=20, entry_tid=48405' 2026-02-19T20:38:54.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:54.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:54.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:54.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:54.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=48405' 2026-02-19T20:38:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:54.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:54.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:54.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:54.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:54.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=48405' 2026-02-19T20:38:54.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:54.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:54.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48405 2026-02-19T20:38:54.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 48405 2026-02-19T20:38:54.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:54.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:54.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:54.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:54.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:54.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:54.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:54.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:54.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:54.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:54.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=48702], [object_number=297, tag_tid=20, entry_tid=48701], [object_number=296, tag_tid=20, entry_tid=48700], [object_number=299, tag_tid=20, entry_tid=48699]]], state=connected] 2026-02-19T20:38:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:54.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=48702], [object_number=297, tag_tid=20, entry_tid=48701], [object_number=296, tag_tid=20, entry_tid=48700], [object_number=299, tag_tid=20, entry_tid=48699]]], state=connected] 2026-02-19T20:38:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=298, tag_tid=20, entry_tid=48702' 2026-02-19T20:38:54.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=298, tag_tid=20, entry_tid=48702' 2026-02-19T20:38:54.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=48702' '!=' 'object_number=297, tag_tid=20, entry_tid=48405' 2026-02-19T20:38:54.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:54.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=48702' = 'object_number=297, tag_tid=20, entry_tid=48405' 2026-02-19T20:38:54.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=298, tag_tid=20, entry_tid=48702' 2026-02-19T20:38:54.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:54.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:54.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:54.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:54.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=48702' 2026-02-19T20:38:54.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:54.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=48702' 2026-02-19T20:38:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48702 2026-02-19T20:38:54.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 48702 2026-02-19T20:38:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:54.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:54.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:54.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:54.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:54.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:54.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:54.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:54.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:54.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:54.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:54.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:54.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:54.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:54.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:54.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:54.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:54.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:54.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:54.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=49014], [object_number=297, tag_tid=20, entry_tid=49013], [object_number=296, tag_tid=20, entry_tid=49012], [object_number=299, tag_tid=20, entry_tid=49011]]], state=connected] 2026-02-19T20:38:54.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:54.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:54.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:54.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:55.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:55.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:55.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:55.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=49014], [object_number=297, tag_tid=20, entry_tid=49013], [object_number=296, tag_tid=20, entry_tid=49012], [object_number=299, tag_tid=20, entry_tid=49011]]], state=connected] 2026-02-19T20:38:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=298, tag_tid=20, entry_tid=49014' 2026-02-19T20:38:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=298, tag_tid=20, entry_tid=49014' 2026-02-19T20:38:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=49014' '!=' 'object_number=298, tag_tid=20, entry_tid=48702' 2026-02-19T20:38:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=49014' = 'object_number=298, tag_tid=20, entry_tid=48702' 2026-02-19T20:38:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=298, tag_tid=20, entry_tid=49014' 2026-02-19T20:38:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:55.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=49014' 2026-02-19T20:38:55.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:55.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:55.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=49014' 2026-02-19T20:38:55.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:55.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=49014 2026-02-19T20:38:55.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 49014 2026-02-19T20:38:55.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:55.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:55.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:55.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:55.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:55.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:55.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:55.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:55.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:55.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:55.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:55.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:55.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=49304], [object_number=299, tag_tid=20, entry_tid=49303], [object_number=298, tag_tid=20, entry_tid=49302], [object_number=297, tag_tid=20, entry_tid=49301]]], state=connected] 2026-02-19T20:38:55.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:55.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:55.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:55.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:55.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=49304], [object_number=299, tag_tid=20, entry_tid=49303], [object_number=298, tag_tid=20, entry_tid=49302], [object_number=297, tag_tid=20, entry_tid=49301]]], state=connected] 2026-02-19T20:38:55.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=296, tag_tid=20, entry_tid=49304' 2026-02-19T20:38:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=296, tag_tid=20, entry_tid=49304' 2026-02-19T20:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=49304' '!=' 'object_number=298, tag_tid=20, entry_tid=49014' 2026-02-19T20:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=49304' = 'object_number=298, tag_tid=20, entry_tid=49014' 2026-02-19T20:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=296, tag_tid=20, entry_tid=49304' 2026-02-19T20:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=49304' 2026-02-19T20:38:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:55.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=49304' 2026-02-19T20:38:55.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:55.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=49304 2026-02-19T20:38:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 49304 2026-02-19T20:38:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:55.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:55.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:55.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:55.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:55.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:55.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:55.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:55.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:55.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:55.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:55.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:55.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:55.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:55.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:55.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:55.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:55.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:55.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=49585], [object_number=296, tag_tid=20, entry_tid=49584], [object_number=299, tag_tid=20, entry_tid=49583], [object_number=298, tag_tid=20, entry_tid=49582]]], state=connected] 2026-02-19T20:38:55.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:55.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:55.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:55.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:55.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:55.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:55.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=49585], [object_number=296, tag_tid=20, entry_tid=49584], [object_number=299, tag_tid=20, entry_tid=49583], [object_number=298, tag_tid=20, entry_tid=49582]]], state=connected] 2026-02-19T20:38:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=297, tag_tid=20, entry_tid=49585' 2026-02-19T20:38:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=297, tag_tid=20, entry_tid=49585' 2026-02-19T20:38:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=49585' '!=' 'object_number=296, tag_tid=20, entry_tid=49304' 2026-02-19T20:38:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=49585' = 'object_number=296, tag_tid=20, entry_tid=49304' 2026-02-19T20:38:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=297, tag_tid=20, entry_tid=49585' 2026-02-19T20:38:55.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:55.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:55.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=49585' 2026-02-19T20:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=49585' 2026-02-19T20:38:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:55.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=49585 2026-02-19T20:38:55.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 49585 2026-02-19T20:38:55.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:55.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:55.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:56.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:56.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:56.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:56.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:56.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:56.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:56.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=49858], [object_number=297, tag_tid=20, entry_tid=49857], [object_number=296, tag_tid=20, entry_tid=49856], [object_number=299, tag_tid=20, entry_tid=49855]]], state=connected] 2026-02-19T20:38:56.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:56.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:56.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=49858], [object_number=297, tag_tid=20, entry_tid=49857], [object_number=296, tag_tid=20, entry_tid=49856], [object_number=299, tag_tid=20, entry_tid=49855]]], state=connected] 2026-02-19T20:38:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=298, tag_tid=20, entry_tid=49858' 2026-02-19T20:38:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=298, tag_tid=20, entry_tid=49858' 2026-02-19T20:38:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=49858' '!=' 'object_number=297, tag_tid=20, entry_tid=49585' 2026-02-19T20:38:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=49858' = 'object_number=297, tag_tid=20, entry_tid=49585' 2026-02-19T20:38:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=298, tag_tid=20, entry_tid=49858' 2026-02-19T20:38:56.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:56.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=49858' 2026-02-19T20:38:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:56.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:56.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:56.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:56.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:56.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:56.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=49858' 2026-02-19T20:38:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=49858 2026-02-19T20:38:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 49858 2026-02-19T20:38:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:56.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:56.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:56.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:56.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:56.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:56.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:56.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:56.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:56.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:56.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:56.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:56.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:56.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:56.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:56.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:56.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:56.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:56.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:56.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:56.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:56.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=50124], [object_number=299, tag_tid=20, entry_tid=50123], [object_number=298, tag_tid=20, entry_tid=50122], [object_number=297, tag_tid=20, entry_tid=50121]]], state=connected] 2026-02-19T20:38:56.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:56.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:56.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:56.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=50124], [object_number=299, tag_tid=20, entry_tid=50123], [object_number=298, tag_tid=20, entry_tid=50122], [object_number=297, tag_tid=20, entry_tid=50121]]], state=connected] 2026-02-19T20:38:56.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=296, tag_tid=20, entry_tid=50124' 2026-02-19T20:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=296, tag_tid=20, entry_tid=50124' 2026-02-19T20:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=50124' '!=' 'object_number=298, tag_tid=20, entry_tid=49858' 2026-02-19T20:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=50124' = 'object_number=298, tag_tid=20, entry_tid=49858' 2026-02-19T20:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=296, tag_tid=20, entry_tid=50124' 2026-02-19T20:38:56.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:56.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:56.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:56.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=50124' 2026-02-19T20:38:56.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:56.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:56.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:56.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:56.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:56.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=50124' 2026-02-19T20:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=50124 2026-02-19T20:38:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 50124 2026-02-19T20:38:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:56.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:57.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:57.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:57.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:57.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:57.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:57.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:57.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:57.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=50436], [object_number=299, tag_tid=20, entry_tid=50435], [object_number=298, tag_tid=20, entry_tid=50434], [object_number=297, tag_tid=20, entry_tid=50433]]], state=connected] 2026-02-19T20:38:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:57.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:57.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:57.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:57.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=50436], [object_number=299, tag_tid=20, entry_tid=50435], [object_number=298, tag_tid=20, entry_tid=50434], [object_number=297, tag_tid=20, entry_tid=50433]]], state=connected] 2026-02-19T20:38:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=296, tag_tid=20, entry_tid=50436' 2026-02-19T20:38:57.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=296, tag_tid=20, entry_tid=50436' 2026-02-19T20:38:57.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=50436' '!=' 'object_number=296, tag_tid=20, entry_tid=50124' 2026-02-19T20:38:57.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:57.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=50436' = 'object_number=296, tag_tid=20, entry_tid=50124' 2026-02-19T20:38:57.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=296, tag_tid=20, entry_tid=50436' 2026-02-19T20:38:57.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:57.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:57.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:57.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:57.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=50436' 2026-02-19T20:38:57.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:57.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=50436' 2026-02-19T20:38:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=50436 2026-02-19T20:38:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 50436 2026-02-19T20:38:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:57.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:57.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:57.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:57.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:57.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:57.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:57.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:57.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:57.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:57.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:57.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:57.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:57.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:57.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:57.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:57.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:57.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:57.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:57.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:57.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:57.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=50730], [object_number=297, tag_tid=20, entry_tid=50729], [object_number=296, tag_tid=20, entry_tid=50728], [object_number=299, tag_tid=20, entry_tid=50727]]], state=connected] 2026-02-19T20:38:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:57.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:57.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:57.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:57.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:57.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:57.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=298, tag_tid=20, entry_tid=50730], [object_number=297, tag_tid=20, entry_tid=50729], [object_number=296, tag_tid=20, entry_tid=50728], [object_number=299, tag_tid=20, entry_tid=50727]]], state=connected] 2026-02-19T20:38:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=298, tag_tid=20, entry_tid=50730' 2026-02-19T20:38:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=298, tag_tid=20, entry_tid=50730' 2026-02-19T20:38:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=50730' '!=' 'object_number=296, tag_tid=20, entry_tid=50436' 2026-02-19T20:38:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=298, tag_tid=20, entry_tid=50730' = 'object_number=296, tag_tid=20, entry_tid=50436' 2026-02-19T20:38:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=298, tag_tid=20, entry_tid=50730' 2026-02-19T20:38:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=50730' 2026-02-19T20:38:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:57.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:57.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:57.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:57.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:57.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=298, tag_tid=20, entry_tid=50730' 2026-02-19T20:38:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:57.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=50730 2026-02-19T20:38:57.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 50730 2026-02-19T20:38:57.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:57.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:57.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:57.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:57.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:57.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:57.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:57.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:57.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:57.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:57.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:57.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:57.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:57.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=51029], [object_number=296, tag_tid=20, entry_tid=51028], [object_number=299, tag_tid=20, entry_tid=51027], [object_number=298, tag_tid=20, entry_tid=51026]]], state=connected] 2026-02-19T20:38:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:58.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=297, tag_tid=20, entry_tid=51029], [object_number=296, tag_tid=20, entry_tid=51028], [object_number=299, tag_tid=20, entry_tid=51027], [object_number=298, tag_tid=20, entry_tid=51026]]], state=connected] 2026-02-19T20:38:58.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=297, tag_tid=20, entry_tid=51029' 2026-02-19T20:38:58.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=297, tag_tid=20, entry_tid=51029' 2026-02-19T20:38:58.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=51029' '!=' 'object_number=298, tag_tid=20, entry_tid=50730' 2026-02-19T20:38:58.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:58.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=297, tag_tid=20, entry_tid=51029' = 'object_number=298, tag_tid=20, entry_tid=50730' 2026-02-19T20:38:58.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=297, tag_tid=20, entry_tid=51029' 2026-02-19T20:38:58.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:58.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:58.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=51029' 2026-02-19T20:38:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:58.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:58.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=297, tag_tid=20, entry_tid=51029' 2026-02-19T20:38:58.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:58.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=51029 2026-02-19T20:38:58.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 51029 2026-02-19T20:38:58.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:58.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:58.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:58.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:58.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:58.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:58.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:58.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:58.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:58.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:58.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:58.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:58.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:58.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:58.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:58.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:58.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:58.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=51311], [object_number=298, tag_tid=20, entry_tid=51310], [object_number=297, tag_tid=20, entry_tid=51309], [object_number=296, tag_tid=20, entry_tid=51308]]], state=connected] 2026-02-19T20:38:58.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:58.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:58.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:58.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:58.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=299, tag_tid=20, entry_tid=51311], [object_number=298, tag_tid=20, entry_tid=51310], [object_number=297, tag_tid=20, entry_tid=51309], [object_number=296, tag_tid=20, entry_tid=51308]]], state=connected] 2026-02-19T20:38:58.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=299, tag_tid=20, entry_tid=51311' 2026-02-19T20:38:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=299, tag_tid=20, entry_tid=51311' 2026-02-19T20:38:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=51311' '!=' 'object_number=297, tag_tid=20, entry_tid=51029' 2026-02-19T20:38:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=299, tag_tid=20, entry_tid=51311' = 'object_number=297, tag_tid=20, entry_tid=51029' 2026-02-19T20:38:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=299, tag_tid=20, entry_tid=51311' 2026-02-19T20:38:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:58.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=51311' 2026-02-19T20:38:58.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:58.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:58.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=299, tag_tid=20, entry_tid=51311' 2026-02-19T20:38:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=51311 2026-02-19T20:38:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 51311 2026-02-19T20:38:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:58.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:58.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:58.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:58.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:58.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:58.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:58.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:58.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:58.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:58.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:58.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:58.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:58.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:58.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:58.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:58.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:58.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:58.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:58.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:58.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:58.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=51584], [object_number=299, tag_tid=20, entry_tid=51583], [object_number=298, tag_tid=20, entry_tid=51582], [object_number=297, tag_tid=20, entry_tid=51581]]], state=connected] 2026-02-19T20:38:58.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:58.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:58.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:58.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 74 2026-02-19T20:38:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=296, tag_tid=20, entry_tid=51584], [object_number=299, tag_tid=20, entry_tid=51583], [object_number=298, tag_tid=20, entry_tid=51582], [object_number=297, tag_tid=20, entry_tid=51581]]], state=connected] 2026-02-19T20:38:58.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:58.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=296, tag_tid=20, entry_tid=51584' 2026-02-19T20:38:58.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=296, tag_tid=20, entry_tid=51584' 2026-02-19T20:38:58.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=51584' '!=' 'object_number=299, tag_tid=20, entry_tid=51311' 2026-02-19T20:38:58.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:58.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=296, tag_tid=20, entry_tid=51584' = 'object_number=299, tag_tid=20, entry_tid=51311' 2026-02-19T20:38:58.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=296, tag_tid=20, entry_tid=51584' 2026-02-19T20:38:58.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:58.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:58.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=51584' 2026-02-19T20:38:58.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:58.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:58.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=296, tag_tid=20, entry_tid=51584' 2026-02-19T20:38:58.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:58.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:58.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=51584 2026-02-19T20:38:58.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 51584 2026-02-19T20:38:58.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:58.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:58.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:59.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:59.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:59.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:59.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:59.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:59.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:59.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:59.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:38:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=51772], [object_number=303, tag_tid=20, entry_tid=51771], [object_number=302, tag_tid=20, entry_tid=51770], [object_number=301, tag_tid=20, entry_tid=51769]]], state=connected] 2026-02-19T20:38:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:38:59.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:59.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:59.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:59.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=51772], [object_number=303, tag_tid=20, entry_tid=51771], [object_number=302, tag_tid=20, entry_tid=51770], [object_number=301, tag_tid=20, entry_tid=51769]]], state=connected] 2026-02-19T20:38:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=51772' 2026-02-19T20:38:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=51772' 2026-02-19T20:38:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=51772' '!=' 'object_number=296, tag_tid=20, entry_tid=51584' 2026-02-19T20:38:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=51772' = 'object_number=296, tag_tid=20, entry_tid=51584' 2026-02-19T20:38:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=51772' 2026-02-19T20:38:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:59.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=51772' 2026-02-19T20:38:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=51772' 2026-02-19T20:38:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=51772 2026-02-19T20:38:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 51772 2026-02-19T20:38:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:59.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:59.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:38:59.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:38:59.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:38:59.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:38:59.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:59.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:38:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:38:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:38:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:38:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:38:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:38:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:38:59.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:38:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:38:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:38:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:38:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:38:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=52030], [object_number=301, tag_tid=20, entry_tid=52029], [object_number=300, tag_tid=20, entry_tid=52028], [object_number=303, tag_tid=20, entry_tid=52027]]], state=connected] 2026-02-19T20:38:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:38:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:38:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:38:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:38:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:38:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:38:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:38:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:38:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:38:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:38:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:38:59.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:38:59.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:38:59.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:38:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=52030], [object_number=301, tag_tid=20, entry_tid=52029], [object_number=300, tag_tid=20, entry_tid=52028], [object_number=303, tag_tid=20, entry_tid=52027]]], state=connected] 2026-02-19T20:38:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:38:59.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=302, tag_tid=20, entry_tid=52030' 2026-02-19T20:38:59.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=302, tag_tid=20, entry_tid=52030' 2026-02-19T20:38:59.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=52030' '!=' 'object_number=300, tag_tid=20, entry_tid=51772' 2026-02-19T20:38:59.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:38:59.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=52030' = 'object_number=300, tag_tid=20, entry_tid=51772' 2026-02-19T20:38:59.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=302, tag_tid=20, entry_tid=52030' 2026-02-19T20:38:59.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:59.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:59.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:38:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=52030' 2026-02-19T20:38:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:38:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:38:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:38:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:59.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:59.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:38:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=52030' 2026-02-19T20:38:59.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:38:59.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:38:59.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=52030 2026-02-19T20:38:59.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 52030 2026-02-19T20:38:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:38:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:38:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:38:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:38:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:38:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:38:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:38:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:38:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:38:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:38:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:38:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:38:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:38:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:38:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:00.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:00.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:00.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:00.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:00.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:00.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:00.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:00.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:00.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=52296], [object_number=303, tag_tid=20, entry_tid=52295], [object_number=302, tag_tid=20, entry_tid=52294], [object_number=301, tag_tid=20, entry_tid=52293]]], state=connected] 2026-02-19T20:39:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:00.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:00.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:00.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:00.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:00.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:00.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:00.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:00.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=52296], [object_number=303, tag_tid=20, entry_tid=52295], [object_number=302, tag_tid=20, entry_tid=52294], [object_number=301, tag_tid=20, entry_tid=52293]]], state=connected] 2026-02-19T20:39:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=52296' 2026-02-19T20:39:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=52296' 2026-02-19T20:39:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=52296' '!=' 'object_number=302, tag_tid=20, entry_tid=52030' 2026-02-19T20:39:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=52296' = 'object_number=302, tag_tid=20, entry_tid=52030' 2026-02-19T20:39:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=52296' 2026-02-19T20:39:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=52296' 2026-02-19T20:39:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:00.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=52296' 2026-02-19T20:39:00.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:00.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:00.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=52296 2026-02-19T20:39:00.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 52296 2026-02-19T20:39:00.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:00.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:00.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:00.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:00.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:00.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:00.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:00.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:00.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:00.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:00.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:00.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:00.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=52458], [object_number=301, tag_tid=20, entry_tid=52457], [object_number=300, tag_tid=20, entry_tid=52456], [object_number=303, tag_tid=20, entry_tid=52455]]], state=connected] 2026-02-19T20:39:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:00.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:00.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:00.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=52458], [object_number=301, tag_tid=20, entry_tid=52457], [object_number=300, tag_tid=20, entry_tid=52456], [object_number=303, tag_tid=20, entry_tid=52455]]], state=connected] 2026-02-19T20:39:01.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=302, tag_tid=20, entry_tid=52458' 2026-02-19T20:39:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=302, tag_tid=20, entry_tid=52458' 2026-02-19T20:39:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=52458' '!=' 'object_number=300, tag_tid=20, entry_tid=52296' 2026-02-19T20:39:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=52458' = 'object_number=300, tag_tid=20, entry_tid=52296' 2026-02-19T20:39:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=302, tag_tid=20, entry_tid=52458' 2026-02-19T20:39:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=52458' 2026-02-19T20:39:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=52458' 2026-02-19T20:39:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=52458 2026-02-19T20:39:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 52458 2026-02-19T20:39:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:01.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:01.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:01.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:01.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:01.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:01.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:01.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:01.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:01.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:01.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=52764], [object_number=303, tag_tid=20, entry_tid=52763], [object_number=302, tag_tid=20, entry_tid=52762], [object_number=301, tag_tid=20, entry_tid=52761]]], state=connected] 2026-02-19T20:39:01.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=52764], [object_number=303, tag_tid=20, entry_tid=52763], [object_number=302, tag_tid=20, entry_tid=52762], [object_number=301, tag_tid=20, entry_tid=52761]]], state=connected] 2026-02-19T20:39:01.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=52764' 2026-02-19T20:39:01.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=52764' 2026-02-19T20:39:01.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=52764' '!=' 'object_number=302, tag_tid=20, entry_tid=52458' 2026-02-19T20:39:01.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=52764' = 'object_number=302, tag_tid=20, entry_tid=52458' 2026-02-19T20:39:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=52764' 2026-02-19T20:39:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:01.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=52764' 2026-02-19T20:39:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=52764' 2026-02-19T20:39:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=52764 2026-02-19T20:39:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 52764 2026-02-19T20:39:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:01.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:01.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:01.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:01.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:01.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:01.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:01.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:01.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:01.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:01.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:01.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:01.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=53042], [object_number=301, tag_tid=20, entry_tid=53041], [object_number=300, tag_tid=20, entry_tid=53040], [object_number=303, tag_tid=20, entry_tid=53039]]], state=connected] 2026-02-19T20:39:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:01.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:01.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:01.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:01.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:01.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=53042], [object_number=301, tag_tid=20, entry_tid=53041], [object_number=300, tag_tid=20, entry_tid=53040], [object_number=303, tag_tid=20, entry_tid=53039]]], state=connected] 2026-02-19T20:39:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=302, tag_tid=20, entry_tid=53042' 2026-02-19T20:39:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=302, tag_tid=20, entry_tid=53042' 2026-02-19T20:39:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=53042' '!=' 'object_number=300, tag_tid=20, entry_tid=52764' 2026-02-19T20:39:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=53042' = 'object_number=300, tag_tid=20, entry_tid=52764' 2026-02-19T20:39:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=302, tag_tid=20, entry_tid=53042' 2026-02-19T20:39:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=53042' 2026-02-19T20:39:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=53042' 2026-02-19T20:39:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:01.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=53042 2026-02-19T20:39:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 53042 2026-02-19T20:39:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:02.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:02.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:02.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:02.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:02.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:02.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:02.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:02.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:02.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=53314], [object_number=301, tag_tid=20, entry_tid=53313], [object_number=300, tag_tid=20, entry_tid=53312], [object_number=303, tag_tid=20, entry_tid=53311]]], state=connected] 2026-02-19T20:39:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:02.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=53314], [object_number=301, tag_tid=20, entry_tid=53313], [object_number=300, tag_tid=20, entry_tid=53312], [object_number=303, tag_tid=20, entry_tid=53311]]], state=connected] 2026-02-19T20:39:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=302, tag_tid=20, entry_tid=53314' 2026-02-19T20:39:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=302, tag_tid=20, entry_tid=53314' 2026-02-19T20:39:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=53314' '!=' 'object_number=302, tag_tid=20, entry_tid=53042' 2026-02-19T20:39:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=53314' = 'object_number=302, tag_tid=20, entry_tid=53042' 2026-02-19T20:39:02.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=302, tag_tid=20, entry_tid=53314' 2026-02-19T20:39:02.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:02.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:02.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=53314' 2026-02-19T20:39:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:02.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:02.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:02.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=53314' 2026-02-19T20:39:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=53314 2026-02-19T20:39:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 53314 2026-02-19T20:39:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:02.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:02.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:02.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:02.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:02.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:02.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:02.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:02.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:02.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:02.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:02.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:02.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:02.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:02.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:02.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:02.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:02.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:02.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:02.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:02.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:02.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=53588], [object_number=303, tag_tid=20, entry_tid=53587], [object_number=302, tag_tid=20, entry_tid=53586], [object_number=301, tag_tid=20, entry_tid=53585]]], state=connected] 2026-02-19T20:39:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=53588], [object_number=303, tag_tid=20, entry_tid=53587], [object_number=302, tag_tid=20, entry_tid=53586], [object_number=301, tag_tid=20, entry_tid=53585]]], state=connected] 2026-02-19T20:39:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:02.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=53588' 2026-02-19T20:39:02.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=53588' 2026-02-19T20:39:02.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=53588' '!=' 'object_number=302, tag_tid=20, entry_tid=53314' 2026-02-19T20:39:02.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:02.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=53588' = 'object_number=302, tag_tid=20, entry_tid=53314' 2026-02-19T20:39:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=53588' 2026-02-19T20:39:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:02.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=53588' 2026-02-19T20:39:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:02.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:02.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:02.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:02.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:02.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=53588' 2026-02-19T20:39:02.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:02.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:02.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=53588 2026-02-19T20:39:02.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 53588 2026-02-19T20:39:02.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:02.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:02.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:02.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:02.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:03.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:03.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=53844], [object_number=303, tag_tid=20, entry_tid=53843], [object_number=302, tag_tid=20, entry_tid=53842], [object_number=301, tag_tid=20, entry_tid=53841]]], state=connected] 2026-02-19T20:39:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=53844], [object_number=303, tag_tid=20, entry_tid=53843], [object_number=302, tag_tid=20, entry_tid=53842], [object_number=301, tag_tid=20, entry_tid=53841]]], state=connected] 2026-02-19T20:39:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=53844' 2026-02-19T20:39:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=53844' 2026-02-19T20:39:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=53844' '!=' 'object_number=300, tag_tid=20, entry_tid=53588' 2026-02-19T20:39:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=53844' = 'object_number=300, tag_tid=20, entry_tid=53588' 2026-02-19T20:39:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=53844' 2026-02-19T20:39:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=53844' 2026-02-19T20:39:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=53844' 2026-02-19T20:39:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=53844 2026-02-19T20:39:03.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 53844 2026-02-19T20:39:03.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:03.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:03.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:03.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:03.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:03.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:03.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:03.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:03.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:03.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:03.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:03.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:03.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:03.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:03.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:03.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=54116], [object_number=303, tag_tid=20, entry_tid=54115], [object_number=302, tag_tid=20, entry_tid=54114], [object_number=301, tag_tid=20, entry_tid=54113]]], state=connected] 2026-02-19T20:39:03.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=54116], [object_number=303, tag_tid=20, entry_tid=54115], [object_number=302, tag_tid=20, entry_tid=54114], [object_number=301, tag_tid=20, entry_tid=54113]]], state=connected] 2026-02-19T20:39:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=54116' 2026-02-19T20:39:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=54116' 2026-02-19T20:39:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=54116' '!=' 'object_number=300, tag_tid=20, entry_tid=53844' 2026-02-19T20:39:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=54116' = 'object_number=300, tag_tid=20, entry_tid=53844' 2026-02-19T20:39:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=54116' 2026-02-19T20:39:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=54116' 2026-02-19T20:39:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:03.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=54116' 2026-02-19T20:39:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=54116 2026-02-19T20:39:03.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 54116 2026-02-19T20:39:03.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:03.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:03.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:03.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:03.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:03.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:03.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:03.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:03.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:03.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:03.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:03.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:03.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=54398], [object_number=301, tag_tid=20, entry_tid=54397], [object_number=300, tag_tid=20, entry_tid=54396], [object_number=303, tag_tid=20, entry_tid=54395]]], state=connected] 2026-02-19T20:39:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:03.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:03.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:03.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:03.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:03.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=302, tag_tid=20, entry_tid=54398], [object_number=301, tag_tid=20, entry_tid=54397], [object_number=300, tag_tid=20, entry_tid=54396], [object_number=303, tag_tid=20, entry_tid=54395]]], state=connected] 2026-02-19T20:39:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=302, tag_tid=20, entry_tid=54398' 2026-02-19T20:39:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=302, tag_tid=20, entry_tid=54398' 2026-02-19T20:39:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=54398' '!=' 'object_number=300, tag_tid=20, entry_tid=54116' 2026-02-19T20:39:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=302, tag_tid=20, entry_tid=54398' = 'object_number=300, tag_tid=20, entry_tid=54116' 2026-02-19T20:39:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=302, tag_tid=20, entry_tid=54398' 2026-02-19T20:39:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=54398' 2026-02-19T20:39:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=302, tag_tid=20, entry_tid=54398' 2026-02-19T20:39:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=54398 2026-02-19T20:39:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 54398 2026-02-19T20:39:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:04.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:04.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:04.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:04.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:04.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:04.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:04.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:04.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:04.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:04.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:04.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:04.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:04.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:04.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:04.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=54656], [object_number=303, tag_tid=20, entry_tid=54655], [object_number=302, tag_tid=20, entry_tid=54654], [object_number=301, tag_tid=20, entry_tid=54653]]], state=connected] 2026-02-19T20:39:04.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:04.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=54656], [object_number=303, tag_tid=20, entry_tid=54655], [object_number=302, tag_tid=20, entry_tid=54654], [object_number=301, tag_tid=20, entry_tid=54653]]], state=connected] 2026-02-19T20:39:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=54656' 2026-02-19T20:39:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=54656' 2026-02-19T20:39:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=54656' '!=' 'object_number=302, tag_tid=20, entry_tid=54398' 2026-02-19T20:39:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=54656' = 'object_number=302, tag_tid=20, entry_tid=54398' 2026-02-19T20:39:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=54656' 2026-02-19T20:39:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=54656' 2026-02-19T20:39:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=54656' 2026-02-19T20:39:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=54656 2026-02-19T20:39:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 54656 2026-02-19T20:39:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:04.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:04.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:04.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=54920], [object_number=303, tag_tid=20, entry_tid=54919], [object_number=302, tag_tid=20, entry_tid=54918], [object_number=301, tag_tid=20, entry_tid=54917]]], state=connected] 2026-02-19T20:39:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:04.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=54920], [object_number=303, tag_tid=20, entry_tid=54919], [object_number=302, tag_tid=20, entry_tid=54918], [object_number=301, tag_tid=20, entry_tid=54917]]], state=connected] 2026-02-19T20:39:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=54920' 2026-02-19T20:39:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=54920' 2026-02-19T20:39:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=54920' '!=' 'object_number=300, tag_tid=20, entry_tid=54656' 2026-02-19T20:39:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=54920' = 'object_number=300, tag_tid=20, entry_tid=54656' 2026-02-19T20:39:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=54920' 2026-02-19T20:39:04.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:04.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:04.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=54920' 2026-02-19T20:39:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:04.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:04.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=54920' 2026-02-19T20:39:04.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:04.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:04.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=54920 2026-02-19T20:39:04.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 54920 2026-02-19T20:39:04.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:04.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:04.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:05.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=55188], [object_number=303, tag_tid=20, entry_tid=55187], [object_number=302, tag_tid=20, entry_tid=55186], [object_number=301, tag_tid=20, entry_tid=55185]]], state=connected] 2026-02-19T20:39:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:05.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=300, tag_tid=20, entry_tid=55188], [object_number=303, tag_tid=20, entry_tid=55187], [object_number=302, tag_tid=20, entry_tid=55186], [object_number=301, tag_tid=20, entry_tid=55185]]], state=connected] 2026-02-19T20:39:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=300, tag_tid=20, entry_tid=55188' 2026-02-19T20:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=300, tag_tid=20, entry_tid=55188' 2026-02-19T20:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=55188' '!=' 'object_number=300, tag_tid=20, entry_tid=54920' 2026-02-19T20:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=300, tag_tid=20, entry_tid=55188' = 'object_number=300, tag_tid=20, entry_tid=54920' 2026-02-19T20:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=300, tag_tid=20, entry_tid=55188' 2026-02-19T20:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=55188' 2026-02-19T20:39:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=300, tag_tid=20, entry_tid=55188' 2026-02-19T20:39:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=55188 2026-02-19T20:39:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 55188 2026-02-19T20:39:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:05.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:05.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:05.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:05.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:05.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:05.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:05.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=301, tag_tid=20, entry_tid=55489], [object_number=300, tag_tid=20, entry_tid=55488], [object_number=303, tag_tid=20, entry_tid=55487], [object_number=302, tag_tid=20, entry_tid=55486]]], state=connected] 2026-02-19T20:39:05.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=301, tag_tid=20, entry_tid=55489], [object_number=300, tag_tid=20, entry_tid=55488], [object_number=303, tag_tid=20, entry_tid=55487], [object_number=302, tag_tid=20, entry_tid=55486]]], state=connected] 2026-02-19T20:39:05.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=301, tag_tid=20, entry_tid=55489' 2026-02-19T20:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=301, tag_tid=20, entry_tid=55489' 2026-02-19T20:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=301, tag_tid=20, entry_tid=55489' '!=' 'object_number=300, tag_tid=20, entry_tid=55188' 2026-02-19T20:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=301, tag_tid=20, entry_tid=55489' = 'object_number=300, tag_tid=20, entry_tid=55188' 2026-02-19T20:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=301, tag_tid=20, entry_tid=55489' 2026-02-19T20:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=301, tag_tid=20, entry_tid=55489' 2026-02-19T20:39:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:05.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:05.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:05.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=301, tag_tid=20, entry_tid=55489' 2026-02-19T20:39:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=55489 2026-02-19T20:39:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 55489 2026-02-19T20:39:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:05.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:06.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:06.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=303, tag_tid=20, entry_tid=55771], [object_number=302, tag_tid=20, entry_tid=55770], [object_number=301, tag_tid=20, entry_tid=55769], [object_number=300, tag_tid=20, entry_tid=55768]]], state=connected] 2026-02-19T20:39:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 75 2026-02-19T20:39:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=303, tag_tid=20, entry_tid=55771], [object_number=302, tag_tid=20, entry_tid=55770], [object_number=301, tag_tid=20, entry_tid=55769], [object_number=300, tag_tid=20, entry_tid=55768]]], state=connected] 2026-02-19T20:39:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=303, tag_tid=20, entry_tid=55771' 2026-02-19T20:39:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=303, tag_tid=20, entry_tid=55771' 2026-02-19T20:39:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=303, tag_tid=20, entry_tid=55771' '!=' 'object_number=301, tag_tid=20, entry_tid=55489' 2026-02-19T20:39:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=303, tag_tid=20, entry_tid=55771' = 'object_number=301, tag_tid=20, entry_tid=55489' 2026-02-19T20:39:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=303, tag_tid=20, entry_tid=55771' 2026-02-19T20:39:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=303, tag_tid=20, entry_tid=55771' 2026-02-19T20:39:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=303, tag_tid=20, entry_tid=55771' 2026-02-19T20:39:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=55771 2026-02-19T20:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 55771 2026-02-19T20:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:06.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:06.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:06.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:06.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:06.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:06.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:06.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=55998], [object_number=305, tag_tid=20, entry_tid=55997], [object_number=304, tag_tid=20, entry_tid=55996], [object_number=307, tag_tid=20, entry_tid=55995]]], state=connected] 2026-02-19T20:39:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:06.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:06.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:06.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:06.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=55998], [object_number=305, tag_tid=20, entry_tid=55997], [object_number=304, tag_tid=20, entry_tid=55996], [object_number=307, tag_tid=20, entry_tid=55995]]], state=connected] 2026-02-19T20:39:06.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=55998' 2026-02-19T20:39:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=55998' 2026-02-19T20:39:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=55998' '!=' 'object_number=303, tag_tid=20, entry_tid=55771' 2026-02-19T20:39:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=55998' = 'object_number=303, tag_tid=20, entry_tid=55771' 2026-02-19T20:39:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=55998' 2026-02-19T20:39:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:06.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=55998' 2026-02-19T20:39:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:06.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=55998' 2026-02-19T20:39:06.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:06.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=55998 2026-02-19T20:39:06.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 55998 2026-02-19T20:39:06.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:06.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:06.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:06.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:06.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:06.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:06.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:06.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:06.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=56274], [object_number=305, tag_tid=20, entry_tid=56273], [object_number=304, tag_tid=20, entry_tid=56272], [object_number=307, tag_tid=20, entry_tid=56271]]], state=connected] 2026-02-19T20:39:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:06.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:06.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:06.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:06.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:06.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=56274], [object_number=305, tag_tid=20, entry_tid=56273], [object_number=304, tag_tid=20, entry_tid=56272], [object_number=307, tag_tid=20, entry_tid=56271]]], state=connected] 2026-02-19T20:39:06.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=56274' 2026-02-19T20:39:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=56274' 2026-02-19T20:39:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=56274' '!=' 'object_number=306, tag_tid=20, entry_tid=55998' 2026-02-19T20:39:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=56274' = 'object_number=306, tag_tid=20, entry_tid=55998' 2026-02-19T20:39:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=56274' 2026-02-19T20:39:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=56274' 2026-02-19T20:39:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:06.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:06.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:06.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=56274' 2026-02-19T20:39:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:06.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=56274 2026-02-19T20:39:06.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 56274 2026-02-19T20:39:06.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:06.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:06.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:07.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:07.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:07.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:07.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:07.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:07.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:07.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:07.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:07.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:07.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:07.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:07.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:07.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:07.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:07.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:07.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=56586], [object_number=305, tag_tid=20, entry_tid=56585], [object_number=304, tag_tid=20, entry_tid=56584], [object_number=307, tag_tid=20, entry_tid=56583]]], state=connected] 2026-02-19T20:39:07.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:07.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:07.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:07.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=56586], [object_number=305, tag_tid=20, entry_tid=56585], [object_number=304, tag_tid=20, entry_tid=56584], [object_number=307, tag_tid=20, entry_tid=56583]]], state=connected] 2026-02-19T20:39:07.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=56586' 2026-02-19T20:39:07.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=56586' 2026-02-19T20:39:07.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=56586' '!=' 'object_number=306, tag_tid=20, entry_tid=56274' 2026-02-19T20:39:07.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:07.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=56586' = 'object_number=306, tag_tid=20, entry_tid=56274' 2026-02-19T20:39:07.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=56586' 2026-02-19T20:39:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:07.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=56586' 2026-02-19T20:39:07.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:07.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:07.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:07.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:07.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:07.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=56586' 2026-02-19T20:39:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=56586 2026-02-19T20:39:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 56586 2026-02-19T20:39:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:07.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:07.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:07.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:07.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:07.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:07.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:07.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:07.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:07.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:07.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:07.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:07.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:07.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:07.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=56874], [object_number=305, tag_tid=20, entry_tid=56873], [object_number=304, tag_tid=20, entry_tid=56872], [object_number=307, tag_tid=20, entry_tid=56871]]], state=connected] 2026-02-19T20:39:07.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:07.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:07.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:07.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:07.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:07.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:07.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=306, tag_tid=20, entry_tid=56874], [object_number=305, tag_tid=20, entry_tid=56873], [object_number=304, tag_tid=20, entry_tid=56872], [object_number=307, tag_tid=20, entry_tid=56871]]], state=connected] 2026-02-19T20:39:07.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=306, tag_tid=20, entry_tid=56874' 2026-02-19T20:39:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=306, tag_tid=20, entry_tid=56874' 2026-02-19T20:39:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=56874' '!=' 'object_number=306, tag_tid=20, entry_tid=56586' 2026-02-19T20:39:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=306, tag_tid=20, entry_tid=56874' = 'object_number=306, tag_tid=20, entry_tid=56586' 2026-02-19T20:39:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=306, tag_tid=20, entry_tid=56874' 2026-02-19T20:39:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=56874' 2026-02-19T20:39:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=306, tag_tid=20, entry_tid=56874' 2026-02-19T20:39:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=56874 2026-02-19T20:39:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 56874 2026-02-19T20:39:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:08.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:08.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:08.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:08.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=307, tag_tid=20, entry_tid=57123], [object_number=306, tag_tid=20, entry_tid=57122], [object_number=305, tag_tid=20, entry_tid=57121], [object_number=304, tag_tid=20, entry_tid=57120]]], state=connected] 2026-02-19T20:39:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=307, tag_tid=20, entry_tid=57123], [object_number=306, tag_tid=20, entry_tid=57122], [object_number=305, tag_tid=20, entry_tid=57121], [object_number=304, tag_tid=20, entry_tid=57120]]], state=connected] 2026-02-19T20:39:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=307, tag_tid=20, entry_tid=57123' 2026-02-19T20:39:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=307, tag_tid=20, entry_tid=57123' 2026-02-19T20:39:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=307, tag_tid=20, entry_tid=57123' '!=' 'object_number=306, tag_tid=20, entry_tid=56874' 2026-02-19T20:39:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=307, tag_tid=20, entry_tid=57123' = 'object_number=306, tag_tid=20, entry_tid=56874' 2026-02-19T20:39:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=307, tag_tid=20, entry_tid=57123' 2026-02-19T20:39:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:08.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=307, tag_tid=20, entry_tid=57123' 2026-02-19T20:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:08.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:08.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=307, tag_tid=20, entry_tid=57123' 2026-02-19T20:39:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=57123 2026-02-19T20:39:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 57123 2026-02-19T20:39:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:08.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:08.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:08.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:08.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:08.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:08.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:08.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374], [object_number=305, tag_tid=20, entry_tid=57373], [object_number=304, tag_tid=20, entry_tid=57372]]], state=connected] 2026-02-19T20:39:08.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:08.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:08.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374], [object_number=305, tag_tid=20, entry_tid=57373], [object_number=304, tag_tid=20, entry_tid=57372]]], state=connected] 2026-02-19T20:39:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=307, tag_tid=20, entry_tid=57375' 2026-02-19T20:39:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=307, tag_tid=20, entry_tid=57375' 2026-02-19T20:39:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=307, tag_tid=20, entry_tid=57375' '!=' 'object_number=307, tag_tid=20, entry_tid=57123' 2026-02-19T20:39:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=307, tag_tid=20, entry_tid=57375' = 'object_number=307, tag_tid=20, entry_tid=57123' 2026-02-19T20:39:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=307, tag_tid=20, entry_tid=57375' 2026-02-19T20:39:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:08.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:08.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=21 2026-02-19T20:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=307, tag_tid=20, entry_tid=57375' 2026-02-19T20:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=20 2026-02-19T20:39:08.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:08.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:08.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=307, tag_tid=20, entry_tid=57375' 2026-02-19T20:39:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:08.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=57375 2026-02-19T20:39:08.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 21 = 20 -a 1 -le 57375 2026-02-19T20:39:08.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:08.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:08.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:09.077 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7046962207989421) > weight (0.25) 2026-02-19T20:39:09.078 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.804723437622755) > weight (0.25) 2026-02-19T20:39:09.078 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-19T20:39:09.078 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-19T20:39:09.079 INFO:tasks.rbd_mirror_thrash:waiting for 8.371379589904697 secs before reviving daemons 2026-02-19T20:39:09.079 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T20:39:09.074+0000 7feca77fe640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 19551) UID: 1000 2026-02-19T20:39:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:09.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 76 2026-02-19T20:39:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 76 2026-02-19T20:39:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=305, tag_tid=21, entry_tid=1], [object_number=304, tag_tid=21, entry_tid=0], [object_number=307, tag_tid=20, entry_tid=57375], [object_number=306, tag_tid=20, entry_tid=57374]]], state=connected] 2026-02-19T20:39:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=305, tag_tid=21, entry_tid=1' -a 'object_number=305, tag_tid=21, entry_tid=1' = 'object_number=305, tag_tid=21, entry_tid=1' 2026-02-19T20:39:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2026-02-19T20:39:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap7 2026-02-19T20:39:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:39:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:39:09.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:39:10.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2026-02-19T20:39:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:39:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:39:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:39:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 13 2026-02-19T20:39:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:39:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:39:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:39:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:39:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:39:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:39:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:39:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:37:31 2026 2026-02-19T20:39:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:38:40 2026 2026-02-19T20:39:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:39:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:39:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:39:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:39:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:39:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:39:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n snap6-clean ']' 2026-02-19T20:39:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-19T20:39:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap6-clean 2026-02-19T20:39:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:39:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:39:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:39:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:39:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:39:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:39:10.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:39:10.999 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:39:10.980+0000 7f9396f28640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-19T20:39:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:39:11.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:39:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:39:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap7 2026-02-19T20:39:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap7 2026-02-19T20:39:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:39:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:39:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:39:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:39:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:39:12.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:39:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:39:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:39:12.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:39:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:39:13.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name=snap7-clean 2026-02-19T20:39:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap7-clean 2026-02-19T20:39:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:39:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap7-clean 2026-02-19T20:39:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2026-02-19T20:39:13.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:39:13.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror test 2026-02-19T20:39:13.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:39:13.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:13.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:13.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:39:13.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=8 2026-02-19T20:39:13.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:39:13.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-19T20:39:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 21936128 bytes 2026-02-19T20:39:14.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 63218688 bytes 2026-02-19T20:39:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 142710784 bytes 2026-02-19T20:39:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 155641856 bytes 2026-02-19T20:39:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 173734912 bytes 2026-02-19T20:39:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 201491456 bytes 2026-02-19T20:39:17.451 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-19T20:39:17.451 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:39:17.452 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-19T20:39:17.452 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-19T20:39:17.452 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-19T20:39:17.453 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-19T20:39:17.453 INFO:tasks.rbd_mirror_thrash:waiting for 109.2033647918629 secs before thrashing 2026-02-19T20:39:18.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 243824640 bytes 2026-02-19T20:39:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 319391744 bytes 2026-02-19T20:39:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 356512768 bytes 2026-02-19T20:39:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 377862144 bytes 2026-02-19T20:39:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 391482368 bytes 2026-02-19T20:39:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 392109056 bytes 2026-02-19T20:39:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 467099648 bytes 2026-02-19T20:39:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 491058176 bytes 2026-02-19T20:39:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:39:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:39:22.182 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:39:22.176+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:39:22.183 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:39:22.176+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:39:22.183 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:39:22.176+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00d096c0 2026-02-19T20:39:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-19T20:39:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:39:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:39:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:39:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:39:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:39:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-19T20:39:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='test: 2026-02-19T20:39:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: global_id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:39:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+replaying 2026-02-19T20:39:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: replaying, {"bytes_per_second":8264367.17,"entries_behind_primary":1,"entries_per_second":515.67,"non_primary_position":{"entry_tid":1,"object_number":305,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":305,"tag_tid":22},"seconds_until_synced":0} 2026-02-19T20:39:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: service: mirror.1 on trial151 2026-02-19T20:39:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:39:17 2026-02-19T20:39:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: peer_sites: 2026-02-19T20:39:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: name: cluster2 2026-02-19T20:39:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: down+unknown 2026-02-19T20:39:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: status not found 2026-02-19T20:39:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: ' 2026-02-19T20:39:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ state: .*up+replaying' 2026-02-19T20:39:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: state: up+replaying 2026-02-19T20:39:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ description: .*' 2026-02-19T20:39:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: description: replaying, {"bytes_per_second":8264367.17,"entries_behind_primary":1,"entries_per_second":515.67,"non_primary_position":{"entry_tid":1,"object_number":305,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":305,"tag_tid":22},"seconds_until_synced":0} 2026-02-19T20:39:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:39:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:39:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^up+' 2026-02-19T20:39:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:39:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep 'service: *mirror..* on ' 2026-02-19T20:39:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: service: mirror.1 on trial151 2026-02-19T20:39:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update 2026-02-19T20:39:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-19T20:39:22.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:39:17' 2026-02-19T20:39:22.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-19 20:39:17' 2026-02-19T20:39:22.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:22.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:22.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:22.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:39:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'prev_last_update=2026-02-19 20:39:17' 2026-02-19T20:39:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:39:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:39:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-19T20:39:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='WARNINGOKWARNING142064262mirror.1trial15120.3.0-5336-g802bf111falseOK42104271mirror.2trial15120.3.0-5336-g802bf111trueOK43714374mirror.3trial15120.3.0-5336-g802bf111falseOK44244426mirror.0trial15120.3.0-5336-g802bf111falseOKtest959deb13-de1a-48d9-a341-1f95416f8379up+replayingreplaying, {"bytes_per_second":8264367.17,"entries_behind_primary":1,"entries_per_second":515.67,"non_primary_position":{"entry_tid":1,"object_number":305,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":305,"tag_tid":22},"seconds_until_synced":0}42064262mirror.1trial1512026-02-19 20:39:17cluster2a054b997-3cfb-48e6-978a-70d6a85e56dedown+unknownstatus not found' 2026-02-19T20:39:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update state 2026-02-19T20:39:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-19T20:39:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:39:17' 2026-02-19T20:39:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-19T20:39:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=up+replaying 2026-02-19T20:39:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:39:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep up+replaying 2026-02-19T20:39:22.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:39:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap8 2026-02-19T20:39:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap8 2026-02-19T20:39:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:39:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap8 2026-02-19T20:39:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:39:54.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:39:54.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:54.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:54.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:54.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:39:54.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:54.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:54.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:54.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:39:54.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:39:54.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:54.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:39:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:39:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:39:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:39:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:39:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:39:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:39:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:55.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:55.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:39:55.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:39:55.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:39:55.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:39:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:55.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:55.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:55.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:55.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:39:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:39:55.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:39:55.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:39:55.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:39:55.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:39:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:39:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:39:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:39:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:39:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:39:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:39:55.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:55.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:55.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:56.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:56.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 81 2026-02-19T20:39:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=326, tag_tid=23, entry_tid=23178], [object_number=325, tag_tid=23, entry_tid=23177], [object_number=324, tag_tid=23, entry_tid=23176], [object_number=327, tag_tid=23, entry_tid=23175]]], state=connected] 2026-02-19T20:39:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:56.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:56.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:56.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:56.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:56.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:56.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:56.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:56.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:56.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:56.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:56.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:56.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:56.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:56.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 81 2026-02-19T20:39:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:56.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=326, tag_tid=23, entry_tid=23178], [object_number=325, tag_tid=23, entry_tid=23177], [object_number=324, tag_tid=23, entry_tid=23176], [object_number=327, tag_tid=23, entry_tid=23175]]], state=connected] 2026-02-19T20:39:56.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=326, tag_tid=23, entry_tid=23178' 2026-02-19T20:39:56.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=326, tag_tid=23, entry_tid=23178' 2026-02-19T20:39:56.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=326, tag_tid=23, entry_tid=23178' '!=' '' 2026-02-19T20:39:56.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:56.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=326, tag_tid=23, entry_tid=23178' = '' 2026-02-19T20:39:56.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=326, tag_tid=23, entry_tid=23178' 2026-02-19T20:39:56.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:56.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:56.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:56.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:39:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=326, tag_tid=23, entry_tid=23178' 2026-02-19T20:39:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:56.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:39:56.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:56.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:56.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=326, tag_tid=23, entry_tid=23178' 2026-02-19T20:39:56.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:56.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:56.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23178 2026-02-19T20:39:56.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 23178 2026-02-19T20:39:56.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:56.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:56.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:56.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:56.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:56.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:56.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:56.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:56.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:56.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:56.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:56.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:56.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:56.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:56.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:56.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:56.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:56.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 81 2026-02-19T20:39:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:56.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:56.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=324, tag_tid=23, entry_tid=23448], [object_number=327, tag_tid=23, entry_tid=23447], [object_number=326, tag_tid=23, entry_tid=23446], [object_number=325, tag_tid=23, entry_tid=23445]]], state=connected] 2026-02-19T20:39:56.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:56.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:56.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 81 2026-02-19T20:39:56.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:56.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:56.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:56.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=324, tag_tid=23, entry_tid=23448], [object_number=327, tag_tid=23, entry_tid=23447], [object_number=326, tag_tid=23, entry_tid=23446], [object_number=325, tag_tid=23, entry_tid=23445]]], state=connected] 2026-02-19T20:39:56.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:56.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=324, tag_tid=23, entry_tid=23448' 2026-02-19T20:39:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=324, tag_tid=23, entry_tid=23448' 2026-02-19T20:39:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=324, tag_tid=23, entry_tid=23448' '!=' 'object_number=326, tag_tid=23, entry_tid=23178' 2026-02-19T20:39:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=324, tag_tid=23, entry_tid=23448' = 'object_number=326, tag_tid=23, entry_tid=23178' 2026-02-19T20:39:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=324, tag_tid=23, entry_tid=23448' 2026-02-19T20:39:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:56.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:39:56.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=324, tag_tid=23, entry_tid=23448' 2026-02-19T20:39:56.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:56.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:56.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:39:56.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:56.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:56.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:56.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=324, tag_tid=23, entry_tid=23448' 2026-02-19T20:39:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23448 2026-02-19T20:39:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 23448 2026-02-19T20:39:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:56.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:56.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:56.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:56.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:56.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:56.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:56.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:56.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:56.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:56.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:56.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:56.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:57.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:57.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:57.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:57.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:57.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:57.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:57.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=330, tag_tid=23, entry_tid=23654], [object_number=329, tag_tid=23, entry_tid=23653], [object_number=328, tag_tid=23, entry_tid=23652], [object_number=331, tag_tid=23, entry_tid=23651]]], state=connected] 2026-02-19T20:39:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=330, tag_tid=23, entry_tid=23654], [object_number=329, tag_tid=23, entry_tid=23653], [object_number=328, tag_tid=23, entry_tid=23652], [object_number=331, tag_tid=23, entry_tid=23651]]], state=connected] 2026-02-19T20:39:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=330, tag_tid=23, entry_tid=23654' 2026-02-19T20:39:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=330, tag_tid=23, entry_tid=23654' 2026-02-19T20:39:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=330, tag_tid=23, entry_tid=23654' '!=' 'object_number=324, tag_tid=23, entry_tid=23448' 2026-02-19T20:39:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=330, tag_tid=23, entry_tid=23654' = 'object_number=324, tag_tid=23, entry_tid=23448' 2026-02-19T20:39:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=330, tag_tid=23, entry_tid=23654' 2026-02-19T20:39:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:39:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=330, tag_tid=23, entry_tid=23654' 2026-02-19T20:39:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:39:57.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:57.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:57.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:57.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:57.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=330, tag_tid=23, entry_tid=23654' 2026-02-19T20:39:57.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:57.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23654 2026-02-19T20:39:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 23654 2026-02-19T20:39:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:57.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:57.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:57.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:57.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:57.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:57.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:57.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:57.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:57.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:57.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:57.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:57.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:57.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:57.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:57.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:57.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:57.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:57.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:57.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:57.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:57.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=330, tag_tid=23, entry_tid=23910], [object_number=329, tag_tid=23, entry_tid=23909], [object_number=328, tag_tid=23, entry_tid=23908], [object_number=331, tag_tid=23, entry_tid=23907]]], state=connected] 2026-02-19T20:39:57.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:57.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:57.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:57.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:57.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:57.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:57.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:57.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=330, tag_tid=23, entry_tid=23910], [object_number=329, tag_tid=23, entry_tid=23909], [object_number=328, tag_tid=23, entry_tid=23908], [object_number=331, tag_tid=23, entry_tid=23907]]], state=connected] 2026-02-19T20:39:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=330, tag_tid=23, entry_tid=23910' 2026-02-19T20:39:57.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=330, tag_tid=23, entry_tid=23910' 2026-02-19T20:39:57.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=330, tag_tid=23, entry_tid=23910' '!=' 'object_number=330, tag_tid=23, entry_tid=23654' 2026-02-19T20:39:57.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:57.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=330, tag_tid=23, entry_tid=23910' = 'object_number=330, tag_tid=23, entry_tid=23654' 2026-02-19T20:39:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=330, tag_tid=23, entry_tid=23910' 2026-02-19T20:39:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:39:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=330, tag_tid=23, entry_tid=23910' 2026-02-19T20:39:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:39:57.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:57.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:57.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:57.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=330, tag_tid=23, entry_tid=23910' 2026-02-19T20:39:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23910 2026-02-19T20:39:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 23910 2026-02-19T20:39:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:57.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:57.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:57.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:57.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:57.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:57.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:57.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:57.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:57.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:57.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:57.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:57.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:57.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:57.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:57.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:57.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:57.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:57.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=331, tag_tid=23, entry_tid=24187], [object_number=330, tag_tid=23, entry_tid=24186], [object_number=329, tag_tid=23, entry_tid=24185], [object_number=328, tag_tid=23, entry_tid=24184]]], state=connected] 2026-02-19T20:39:57.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:57.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:57.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:57.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:57.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:57.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:57.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:57.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:57.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:57.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:57.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:57.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=331, tag_tid=23, entry_tid=24187], [object_number=330, tag_tid=23, entry_tid=24186], [object_number=329, tag_tid=23, entry_tid=24185], [object_number=328, tag_tid=23, entry_tid=24184]]], state=connected] 2026-02-19T20:39:58.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=331, tag_tid=23, entry_tid=24187' 2026-02-19T20:39:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=331, tag_tid=23, entry_tid=24187' 2026-02-19T20:39:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=331, tag_tid=23, entry_tid=24187' '!=' 'object_number=330, tag_tid=23, entry_tid=23910' 2026-02-19T20:39:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=331, tag_tid=23, entry_tid=24187' = 'object_number=330, tag_tid=23, entry_tid=23910' 2026-02-19T20:39:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=331, tag_tid=23, entry_tid=24187' 2026-02-19T20:39:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:39:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=331, tag_tid=23, entry_tid=24187' 2026-02-19T20:39:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:39:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=331, tag_tid=23, entry_tid=24187' 2026-02-19T20:39:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24187 2026-02-19T20:39:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 24187 2026-02-19T20:39:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:58.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:58.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:58.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:58.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:58.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:58.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:58.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:58.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:58.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:58.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:58.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:58.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:58.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:58.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:58.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:58.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:58.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:58.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:58.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=329, tag_tid=23, entry_tid=24485], [object_number=328, tag_tid=23, entry_tid=24484], [object_number=331, tag_tid=23, entry_tid=24483], [object_number=330, tag_tid=23, entry_tid=24482]]], state=connected] 2026-02-19T20:39:58.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:58.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:58.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:58.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:58.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:58.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:58.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:58.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:58.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:58.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:58.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:58.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:58.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:58.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:58.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:58.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:58.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:58.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:58.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=329, tag_tid=23, entry_tid=24485], [object_number=328, tag_tid=23, entry_tid=24484], [object_number=331, tag_tid=23, entry_tid=24483], [object_number=330, tag_tid=23, entry_tid=24482]]], state=connected] 2026-02-19T20:39:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=329, tag_tid=23, entry_tid=24485' 2026-02-19T20:39:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=329, tag_tid=23, entry_tid=24485' 2026-02-19T20:39:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=329, tag_tid=23, entry_tid=24485' '!=' 'object_number=331, tag_tid=23, entry_tid=24187' 2026-02-19T20:39:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=329, tag_tid=23, entry_tid=24485' = 'object_number=331, tag_tid=23, entry_tid=24187' 2026-02-19T20:39:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=329, tag_tid=23, entry_tid=24485' 2026-02-19T20:39:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:39:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=329, tag_tid=23, entry_tid=24485' 2026-02-19T20:39:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:39:58.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:58.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:58.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:58.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=329, tag_tid=23, entry_tid=24485' 2026-02-19T20:39:58.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:58.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24485 2026-02-19T20:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 24485 2026-02-19T20:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:58.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:58.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:58.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:58.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:58.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:58.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:58.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:58.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:58.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:58.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:58.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:58.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:58.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:58.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:58.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:58.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=24720], [object_number=331, tag_tid=23, entry_tid=24719], [object_number=330, tag_tid=23, entry_tid=24718], [object_number=329, tag_tid=23, entry_tid=24717]]], state=connected] 2026-02-19T20:39:58.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:58.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:58.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:58.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=24720], [object_number=331, tag_tid=23, entry_tid=24719], [object_number=330, tag_tid=23, entry_tid=24718], [object_number=329, tag_tid=23, entry_tid=24717]]], state=connected] 2026-02-19T20:39:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=328, tag_tid=23, entry_tid=24720' 2026-02-19T20:39:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=328, tag_tid=23, entry_tid=24720' 2026-02-19T20:39:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=24720' '!=' 'object_number=329, tag_tid=23, entry_tid=24485' 2026-02-19T20:39:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=24720' = 'object_number=329, tag_tid=23, entry_tid=24485' 2026-02-19T20:39:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=328, tag_tid=23, entry_tid=24720' 2026-02-19T20:39:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:39:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=24720' 2026-02-19T20:39:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:39:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=24720' 2026-02-19T20:39:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24720 2026-02-19T20:39:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 24720 2026-02-19T20:39:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:59.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:59.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:59.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:59.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:59.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:59.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:59.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:59.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:59.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=24984], [object_number=331, tag_tid=23, entry_tid=24983], [object_number=330, tag_tid=23, entry_tid=24982], [object_number=329, tag_tid=23, entry_tid=24981]]], state=connected] 2026-02-19T20:39:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:59.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:59.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:59.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:59.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:59.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=24984], [object_number=331, tag_tid=23, entry_tid=24983], [object_number=330, tag_tid=23, entry_tid=24982], [object_number=329, tag_tid=23, entry_tid=24981]]], state=connected] 2026-02-19T20:39:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=328, tag_tid=23, entry_tid=24984' 2026-02-19T20:39:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=328, tag_tid=23, entry_tid=24984' 2026-02-19T20:39:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=24984' '!=' 'object_number=328, tag_tid=23, entry_tid=24720' 2026-02-19T20:39:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=24984' = 'object_number=328, tag_tid=23, entry_tid=24720' 2026-02-19T20:39:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=328, tag_tid=23, entry_tid=24984' 2026-02-19T20:39:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:59.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:39:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=24984' 2026-02-19T20:39:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:39:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:59.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=24984' 2026-02-19T20:39:59.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:59.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:59.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24984 2026-02-19T20:39:59.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 24984 2026-02-19T20:39:59.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:59.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:59.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:59.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:59.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:59.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:59.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:59.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:59.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:59.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:39:59.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:39:59.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:39:59.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:39:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:39:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:39:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:39:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:39:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:39:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:39:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:39:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:39:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:39:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:39:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:39:59.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:39:59.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:39:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:39:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:39:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:39:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=331, tag_tid=23, entry_tid=25295], [object_number=330, tag_tid=23, entry_tid=25294], [object_number=329, tag_tid=23, entry_tid=25293], [object_number=328, tag_tid=23, entry_tid=25292]]], state=connected] 2026-02-19T20:39:59.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:39:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:39:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:39:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:39:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:39:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:39:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:39:59.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:39:59.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:39:59.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:39:59.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:39:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:39:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:39:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:39:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=331, tag_tid=23, entry_tid=25295], [object_number=330, tag_tid=23, entry_tid=25294], [object_number=329, tag_tid=23, entry_tid=25293], [object_number=328, tag_tid=23, entry_tid=25292]]], state=connected] 2026-02-19T20:39:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:39:59.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=331, tag_tid=23, entry_tid=25295' 2026-02-19T20:39:59.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=331, tag_tid=23, entry_tid=25295' 2026-02-19T20:39:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=331, tag_tid=23, entry_tid=25295' '!=' 'object_number=328, tag_tid=23, entry_tid=24984' 2026-02-19T20:39:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:39:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=331, tag_tid=23, entry_tid=25295' = 'object_number=328, tag_tid=23, entry_tid=24984' 2026-02-19T20:39:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=331, tag_tid=23, entry_tid=25295' 2026-02-19T20:39:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:39:59.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=331, tag_tid=23, entry_tid=25295' 2026-02-19T20:39:59.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:39:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:39:59.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:39:59.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:59.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:39:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=331, tag_tid=23, entry_tid=25295' 2026-02-19T20:39:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:39:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:39:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25295 2026-02-19T20:39:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 25295 2026-02-19T20:39:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:39:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:39:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:39:59.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:39:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:39:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:39:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:39:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:39:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:39:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:39:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:39:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:39:59.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:39:59.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:39:59.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:39:59.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:39:59.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:39:59.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:39:59.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:39:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:39:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:39:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:39:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:39:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:39:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:39:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:39:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:39:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:39:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:39:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:39:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:39:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:39:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:39:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:39:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:39:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:39:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:39:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:39:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:39:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:39:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:39:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:39:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:39:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:39:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:39:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:39:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:39:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:39:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:00.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:00.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:00.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=329, tag_tid=23, entry_tid=25585], [object_number=328, tag_tid=23, entry_tid=25584], [object_number=331, tag_tid=23, entry_tid=25583], [object_number=330, tag_tid=23, entry_tid=25582]]], state=connected] 2026-02-19T20:40:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=329, tag_tid=23, entry_tid=25585], [object_number=328, tag_tid=23, entry_tid=25584], [object_number=331, tag_tid=23, entry_tid=25583], [object_number=330, tag_tid=23, entry_tid=25582]]], state=connected] 2026-02-19T20:40:00.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=329, tag_tid=23, entry_tid=25585' 2026-02-19T20:40:00.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=329, tag_tid=23, entry_tid=25585' 2026-02-19T20:40:00.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=329, tag_tid=23, entry_tid=25585' '!=' 'object_number=331, tag_tid=23, entry_tid=25295' 2026-02-19T20:40:00.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:00.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=329, tag_tid=23, entry_tid=25585' = 'object_number=331, tag_tid=23, entry_tid=25295' 2026-02-19T20:40:00.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=329, tag_tid=23, entry_tid=25585' 2026-02-19T20:40:00.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:00.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:00.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=329, tag_tid=23, entry_tid=25585' 2026-02-19T20:40:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=329, tag_tid=23, entry_tid=25585' 2026-02-19T20:40:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25585 2026-02-19T20:40:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 25585 2026-02-19T20:40:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:00.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:00.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:00.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:00.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:00.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=330, tag_tid=23, entry_tid=25878], [object_number=329, tag_tid=23, entry_tid=25877], [object_number=328, tag_tid=23, entry_tid=25876], [object_number=331, tag_tid=23, entry_tid=25875]]], state=connected] 2026-02-19T20:40:00.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:00.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:00.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:00.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:00.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:00.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:00.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:00.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:00.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:00.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:00.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:00.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=330, tag_tid=23, entry_tid=25878], [object_number=329, tag_tid=23, entry_tid=25877], [object_number=328, tag_tid=23, entry_tid=25876], [object_number=331, tag_tid=23, entry_tid=25875]]], state=connected] 2026-02-19T20:40:00.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=330, tag_tid=23, entry_tid=25878' 2026-02-19T20:40:00.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=330, tag_tid=23, entry_tid=25878' 2026-02-19T20:40:00.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=330, tag_tid=23, entry_tid=25878' '!=' 'object_number=329, tag_tid=23, entry_tid=25585' 2026-02-19T20:40:00.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:00.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=330, tag_tid=23, entry_tid=25878' = 'object_number=329, tag_tid=23, entry_tid=25585' 2026-02-19T20:40:00.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=330, tag_tid=23, entry_tid=25878' 2026-02-19T20:40:00.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:00.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:00.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:00.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=330, tag_tid=23, entry_tid=25878' 2026-02-19T20:40:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:00.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:00.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=330, tag_tid=23, entry_tid=25878' 2026-02-19T20:40:00.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:00.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25878 2026-02-19T20:40:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 25878 2026-02-19T20:40:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:00.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:00.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=26176], [object_number=331, tag_tid=23, entry_tid=26175], [object_number=330, tag_tid=23, entry_tid=26174], [object_number=329, tag_tid=23, entry_tid=26173]]], state=connected] 2026-02-19T20:40:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:01.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:01.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=26176], [object_number=331, tag_tid=23, entry_tid=26175], [object_number=330, tag_tid=23, entry_tid=26174], [object_number=329, tag_tid=23, entry_tid=26173]]], state=connected] 2026-02-19T20:40:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=328, tag_tid=23, entry_tid=26176' 2026-02-19T20:40:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=328, tag_tid=23, entry_tid=26176' 2026-02-19T20:40:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=26176' '!=' 'object_number=330, tag_tid=23, entry_tid=25878' 2026-02-19T20:40:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=26176' = 'object_number=330, tag_tid=23, entry_tid=25878' 2026-02-19T20:40:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=328, tag_tid=23, entry_tid=26176' 2026-02-19T20:40:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=26176' 2026-02-19T20:40:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:01.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:01.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:01.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=26176' 2026-02-19T20:40:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26176 2026-02-19T20:40:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 26176 2026-02-19T20:40:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:01.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=26464], [object_number=331, tag_tid=23, entry_tid=26463], [object_number=330, tag_tid=23, entry_tid=26462], [object_number=329, tag_tid=23, entry_tid=26461]]], state=connected] 2026-02-19T20:40:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:01.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=26464], [object_number=331, tag_tid=23, entry_tid=26463], [object_number=330, tag_tid=23, entry_tid=26462], [object_number=329, tag_tid=23, entry_tid=26461]]], state=connected] 2026-02-19T20:40:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=328, tag_tid=23, entry_tid=26464' 2026-02-19T20:40:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=328, tag_tid=23, entry_tid=26464' 2026-02-19T20:40:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=26464' '!=' 'object_number=328, tag_tid=23, entry_tid=26176' 2026-02-19T20:40:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=26464' = 'object_number=328, tag_tid=23, entry_tid=26176' 2026-02-19T20:40:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=328, tag_tid=23, entry_tid=26464' 2026-02-19T20:40:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:01.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:01.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=26464' 2026-02-19T20:40:01.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:01.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:01.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:01.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=26464' 2026-02-19T20:40:01.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:01.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26464 2026-02-19T20:40:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 26464 2026-02-19T20:40:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:01.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:01.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:01.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:01.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:01.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:01.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:01.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:01.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:01.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:01.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:01.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:01.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:01.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:01.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:01.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:01.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:01.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:01.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:01.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:01.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:01.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:01.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:01.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:01.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:01.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:01.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:01.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:01.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:01.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:01.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:01.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:01.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:01.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:01.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:01.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:01.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:01.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:01.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:01.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=26760], [object_number=331, tag_tid=23, entry_tid=26759], [object_number=330, tag_tid=23, entry_tid=26758], [object_number=329, tag_tid=23, entry_tid=26757]]], state=connected] 2026-02-19T20:40:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=26760], [object_number=331, tag_tid=23, entry_tid=26759], [object_number=330, tag_tid=23, entry_tid=26758], [object_number=329, tag_tid=23, entry_tid=26757]]], state=connected] 2026-02-19T20:40:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=328, tag_tid=23, entry_tid=26760' 2026-02-19T20:40:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=328, tag_tid=23, entry_tid=26760' 2026-02-19T20:40:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=26760' '!=' 'object_number=328, tag_tid=23, entry_tid=26464' 2026-02-19T20:40:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=26760' = 'object_number=328, tag_tid=23, entry_tid=26464' 2026-02-19T20:40:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=328, tag_tid=23, entry_tid=26760' 2026-02-19T20:40:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=26760' 2026-02-19T20:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=26760' 2026-02-19T20:40:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26760 2026-02-19T20:40:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 26760 2026-02-19T20:40:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:02.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:02.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:02.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:02.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:02.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:02.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:02.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:02.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:02.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:02.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:02.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=329, tag_tid=23, entry_tid=27025], [object_number=328, tag_tid=23, entry_tid=27024], [object_number=331, tag_tid=23, entry_tid=27023], [object_number=330, tag_tid=23, entry_tid=27022]]], state=connected] 2026-02-19T20:40:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:02.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:02.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:02.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:02.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:02.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:02.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:02.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:02.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=329, tag_tid=23, entry_tid=27025], [object_number=328, tag_tid=23, entry_tid=27024], [object_number=331, tag_tid=23, entry_tid=27023], [object_number=330, tag_tid=23, entry_tid=27022]]], state=connected] 2026-02-19T20:40:02.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=329, tag_tid=23, entry_tid=27025' 2026-02-19T20:40:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=329, tag_tid=23, entry_tid=27025' 2026-02-19T20:40:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=329, tag_tid=23, entry_tid=27025' '!=' 'object_number=328, tag_tid=23, entry_tid=26760' 2026-02-19T20:40:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=329, tag_tid=23, entry_tid=27025' = 'object_number=328, tag_tid=23, entry_tid=26760' 2026-02-19T20:40:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=329, tag_tid=23, entry_tid=27025' 2026-02-19T20:40:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:02.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=329, tag_tid=23, entry_tid=27025' 2026-02-19T20:40:02.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:02.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=329, tag_tid=23, entry_tid=27025' 2026-02-19T20:40:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27025 2026-02-19T20:40:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 27025 2026-02-19T20:40:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:02.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:02.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:02.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:02.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:02.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:02.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=27316], [object_number=331, tag_tid=23, entry_tid=27315], [object_number=330, tag_tid=23, entry_tid=27314], [object_number=329, tag_tid=23, entry_tid=27313]]], state=connected] 2026-02-19T20:40:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:02.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:02.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:02.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:02.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:02.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:02.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:02.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=328, tag_tid=23, entry_tid=27316], [object_number=331, tag_tid=23, entry_tid=27315], [object_number=330, tag_tid=23, entry_tid=27314], [object_number=329, tag_tid=23, entry_tid=27313]]], state=connected] 2026-02-19T20:40:02.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:02.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=328, tag_tid=23, entry_tid=27316' 2026-02-19T20:40:02.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=328, tag_tid=23, entry_tid=27316' 2026-02-19T20:40:02.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=27316' '!=' 'object_number=329, tag_tid=23, entry_tid=27025' 2026-02-19T20:40:02.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:02.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=328, tag_tid=23, entry_tid=27316' = 'object_number=329, tag_tid=23, entry_tid=27025' 2026-02-19T20:40:02.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=328, tag_tid=23, entry_tid=27316' 2026-02-19T20:40:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:02.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:02.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=27316' 2026-02-19T20:40:02.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:02.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=328, tag_tid=23, entry_tid=27316' 2026-02-19T20:40:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27316 2026-02-19T20:40:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 27316 2026-02-19T20:40:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:03.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:03.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:03.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:03.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:03.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:03.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:03.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:03.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:03.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:03.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:03.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:03.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:03.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:03.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:03.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=331, tag_tid=23, entry_tid=27611], [object_number=330, tag_tid=23, entry_tid=27610], [object_number=329, tag_tid=23, entry_tid=27609], [object_number=328, tag_tid=23, entry_tid=27608]]], state=connected] 2026-02-19T20:40:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:03.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:03.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:03.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:03.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:03.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:03.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:03.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:03.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:03.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:03.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:03.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:03.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:03.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:03.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 82 2026-02-19T20:40:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=331, tag_tid=23, entry_tid=27611], [object_number=330, tag_tid=23, entry_tid=27610], [object_number=329, tag_tid=23, entry_tid=27609], [object_number=328, tag_tid=23, entry_tid=27608]]], state=connected] 2026-02-19T20:40:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=331, tag_tid=23, entry_tid=27611' 2026-02-19T20:40:03.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=331, tag_tid=23, entry_tid=27611' 2026-02-19T20:40:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=331, tag_tid=23, entry_tid=27611' '!=' 'object_number=328, tag_tid=23, entry_tid=27316' 2026-02-19T20:40:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=331, tag_tid=23, entry_tid=27611' = 'object_number=328, tag_tid=23, entry_tid=27316' 2026-02-19T20:40:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=331, tag_tid=23, entry_tid=27611' 2026-02-19T20:40:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:03.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=331, tag_tid=23, entry_tid=27611' 2026-02-19T20:40:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:03.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:03.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:03.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:03.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:03.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=331, tag_tid=23, entry_tid=27611' 2026-02-19T20:40:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27611 2026-02-19T20:40:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 27611 2026-02-19T20:40:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:03.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:03.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:03.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:03.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:03.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:03.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:03.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:03.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:03.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:03.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:03.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:03.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:03.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:03.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:03.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:03.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:03.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:03.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:03.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:03.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:03.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:03.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:03.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:03.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:03.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:03.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:03.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:03.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:03.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:03.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:03.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:03.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:03.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:03.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=332, tag_tid=23, entry_tid=27816], [object_number=335, tag_tid=23, entry_tid=27815], [object_number=334, tag_tid=23, entry_tid=27814], [object_number=333, tag_tid=23, entry_tid=27813]]], state=connected] 2026-02-19T20:40:03.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:03.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:03.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:03.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:03.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:03.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:03.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=332, tag_tid=23, entry_tid=27816], [object_number=335, tag_tid=23, entry_tid=27815], [object_number=334, tag_tid=23, entry_tid=27814], [object_number=333, tag_tid=23, entry_tid=27813]]], state=connected] 2026-02-19T20:40:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:03.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=332, tag_tid=23, entry_tid=27816' 2026-02-19T20:40:03.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=332, tag_tid=23, entry_tid=27816' 2026-02-19T20:40:03.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=332, tag_tid=23, entry_tid=27816' '!=' 'object_number=331, tag_tid=23, entry_tid=27611' 2026-02-19T20:40:03.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:03.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=332, tag_tid=23, entry_tid=27816' = 'object_number=331, tag_tid=23, entry_tid=27611' 2026-02-19T20:40:03.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=332, tag_tid=23, entry_tid=27816' 2026-02-19T20:40:03.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:03.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:03.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:03.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:03.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=332, tag_tid=23, entry_tid=27816' 2026-02-19T20:40:03.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:03.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:03.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=332, tag_tid=23, entry_tid=27816' 2026-02-19T20:40:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:03.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27816 2026-02-19T20:40:03.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 27816 2026-02-19T20:40:03.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:03.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:03.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:03.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:03.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:03.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=28162], [object_number=333, tag_tid=23, entry_tid=28161], [object_number=332, tag_tid=23, entry_tid=28160], [object_number=335, tag_tid=23, entry_tid=28159]]], state=connected] 2026-02-19T20:40:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:04.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:04.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:04.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=28162], [object_number=333, tag_tid=23, entry_tid=28161], [object_number=332, tag_tid=23, entry_tid=28160], [object_number=335, tag_tid=23, entry_tid=28159]]], state=connected] 2026-02-19T20:40:04.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=28162' 2026-02-19T20:40:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=28162' 2026-02-19T20:40:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=28162' '!=' 'object_number=332, tag_tid=23, entry_tid=27816' 2026-02-19T20:40:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=28162' = 'object_number=332, tag_tid=23, entry_tid=27816' 2026-02-19T20:40:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=28162' 2026-02-19T20:40:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:04.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=28162' 2026-02-19T20:40:04.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:04.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:04.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:04.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:04.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=28162' 2026-02-19T20:40:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28162 2026-02-19T20:40:04.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 28162 2026-02-19T20:40:04.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:04.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:04.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:04.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:04.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:04.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:04.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:04.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:04.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:04.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:04.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:04.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=28477], [object_number=332, tag_tid=23, entry_tid=28476], [object_number=335, tag_tid=23, entry_tid=28475], [object_number=334, tag_tid=23, entry_tid=28474]]], state=connected] 2026-02-19T20:40:04.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:04.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:04.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:04.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=28477], [object_number=332, tag_tid=23, entry_tid=28476], [object_number=335, tag_tid=23, entry_tid=28475], [object_number=334, tag_tid=23, entry_tid=28474]]], state=connected] 2026-02-19T20:40:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=333, tag_tid=23, entry_tid=28477' 2026-02-19T20:40:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=333, tag_tid=23, entry_tid=28477' 2026-02-19T20:40:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=28477' '!=' 'object_number=334, tag_tid=23, entry_tid=28162' 2026-02-19T20:40:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=28477' = 'object_number=334, tag_tid=23, entry_tid=28162' 2026-02-19T20:40:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=333, tag_tid=23, entry_tid=28477' 2026-02-19T20:40:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:04.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=28477' 2026-02-19T20:40:04.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:04.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=28477' 2026-02-19T20:40:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28477 2026-02-19T20:40:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 28477 2026-02-19T20:40:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:04.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=28781], [object_number=332, tag_tid=23, entry_tid=28780], [object_number=335, tag_tid=23, entry_tid=28779], [object_number=334, tag_tid=23, entry_tid=28778]]], state=connected] 2026-02-19T20:40:04.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:04.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:04.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:04.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:04.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:04.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:04.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:04.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:04.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:04.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:04.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:04.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=28781], [object_number=332, tag_tid=23, entry_tid=28780], [object_number=335, tag_tid=23, entry_tid=28779], [object_number=334, tag_tid=23, entry_tid=28778]]], state=connected] 2026-02-19T20:40:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:04.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=333, tag_tid=23, entry_tid=28781' 2026-02-19T20:40:04.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=333, tag_tid=23, entry_tid=28781' 2026-02-19T20:40:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=28781' '!=' 'object_number=333, tag_tid=23, entry_tid=28477' 2026-02-19T20:40:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=28781' = 'object_number=333, tag_tid=23, entry_tid=28477' 2026-02-19T20:40:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=333, tag_tid=23, entry_tid=28781' 2026-02-19T20:40:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=28781' 2026-02-19T20:40:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:04.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=28781' 2026-02-19T20:40:04.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:04.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:04.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28781 2026-02-19T20:40:04.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 28781 2026-02-19T20:40:04.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:04.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:04.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:05.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:05.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:05.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:05.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:05.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:05.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:05.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=29061], [object_number=332, tag_tid=23, entry_tid=29060], [object_number=335, tag_tid=23, entry_tid=29059], [object_number=334, tag_tid=23, entry_tid=29058]]], state=connected] 2026-02-19T20:40:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:05.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:05.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:05.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:05.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:05.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:05.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:05.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:05.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:05.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:05.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:05.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:05.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:05.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:05.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:05.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=29061], [object_number=332, tag_tid=23, entry_tid=29060], [object_number=335, tag_tid=23, entry_tid=29059], [object_number=334, tag_tid=23, entry_tid=29058]]], state=connected] 2026-02-19T20:40:05.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=333, tag_tid=23, entry_tid=29061' 2026-02-19T20:40:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=333, tag_tid=23, entry_tid=29061' 2026-02-19T20:40:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=29061' '!=' 'object_number=333, tag_tid=23, entry_tid=28781' 2026-02-19T20:40:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=29061' = 'object_number=333, tag_tid=23, entry_tid=28781' 2026-02-19T20:40:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=333, tag_tid=23, entry_tid=29061' 2026-02-19T20:40:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=29061' 2026-02-19T20:40:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:05.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=29061' 2026-02-19T20:40:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:05.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29061 2026-02-19T20:40:05.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 29061 2026-02-19T20:40:05.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:05.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:05.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:05.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:05.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:05.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:05.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:05.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:05.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:05.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:05.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:05.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:05.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:05.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:05.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:05.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:05.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:05.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:05.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=332, tag_tid=23, entry_tid=29240], [object_number=335, tag_tid=23, entry_tid=29239], [object_number=334, tag_tid=23, entry_tid=29238], [object_number=333, tag_tid=23, entry_tid=29237]]], state=connected] 2026-02-19T20:40:05.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:05.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=332, tag_tid=23, entry_tid=29240], [object_number=335, tag_tid=23, entry_tid=29239], [object_number=334, tag_tid=23, entry_tid=29238], [object_number=333, tag_tid=23, entry_tid=29237]]], state=connected] 2026-02-19T20:40:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=332, tag_tid=23, entry_tid=29240' 2026-02-19T20:40:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=332, tag_tid=23, entry_tid=29240' 2026-02-19T20:40:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=332, tag_tid=23, entry_tid=29240' '!=' 'object_number=333, tag_tid=23, entry_tid=29061' 2026-02-19T20:40:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=332, tag_tid=23, entry_tid=29240' = 'object_number=333, tag_tid=23, entry_tid=29061' 2026-02-19T20:40:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=332, tag_tid=23, entry_tid=29240' 2026-02-19T20:40:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=332, tag_tid=23, entry_tid=29240' 2026-02-19T20:40:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=332, tag_tid=23, entry_tid=29240' 2026-02-19T20:40:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29240 2026-02-19T20:40:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 29240 2026-02-19T20:40:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:06.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:06.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:06.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:06.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:06.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:06.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:06.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:06.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:06.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:06.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:06.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:06.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=335, tag_tid=23, entry_tid=29507], [object_number=334, tag_tid=23, entry_tid=29506], [object_number=333, tag_tid=23, entry_tid=29505], [object_number=332, tag_tid=23, entry_tid=29504]]], state=connected] 2026-02-19T20:40:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=335, tag_tid=23, entry_tid=29507], [object_number=334, tag_tid=23, entry_tid=29506], [object_number=333, tag_tid=23, entry_tid=29505], [object_number=332, tag_tid=23, entry_tid=29504]]], state=connected] 2026-02-19T20:40:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=335, tag_tid=23, entry_tid=29507' 2026-02-19T20:40:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=335, tag_tid=23, entry_tid=29507' 2026-02-19T20:40:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=335, tag_tid=23, entry_tid=29507' '!=' 'object_number=332, tag_tid=23, entry_tid=29240' 2026-02-19T20:40:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=335, tag_tid=23, entry_tid=29507' = 'object_number=332, tag_tid=23, entry_tid=29240' 2026-02-19T20:40:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=335, tag_tid=23, entry_tid=29507' 2026-02-19T20:40:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=335, tag_tid=23, entry_tid=29507' 2026-02-19T20:40:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=335, tag_tid=23, entry_tid=29507' 2026-02-19T20:40:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29507 2026-02-19T20:40:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 29507 2026-02-19T20:40:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:06.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:06.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:06.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:06.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:06.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:06.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:06.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:06.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:06.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=29821], [object_number=332, tag_tid=23, entry_tid=29820], [object_number=335, tag_tid=23, entry_tid=29819], [object_number=334, tag_tid=23, entry_tid=29818]]], state=connected] 2026-02-19T20:40:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=29821], [object_number=332, tag_tid=23, entry_tid=29820], [object_number=335, tag_tid=23, entry_tid=29819], [object_number=334, tag_tid=23, entry_tid=29818]]], state=connected] 2026-02-19T20:40:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=333, tag_tid=23, entry_tid=29821' 2026-02-19T20:40:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=333, tag_tid=23, entry_tid=29821' 2026-02-19T20:40:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=29821' '!=' 'object_number=335, tag_tid=23, entry_tid=29507' 2026-02-19T20:40:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=29821' = 'object_number=335, tag_tid=23, entry_tid=29507' 2026-02-19T20:40:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=333, tag_tid=23, entry_tid=29821' 2026-02-19T20:40:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=29821' 2026-02-19T20:40:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=29821' 2026-02-19T20:40:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:06.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:06.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29821 2026-02-19T20:40:06.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 29821 2026-02-19T20:40:06.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:06.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:07.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:07.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:07.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:07.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:07.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:07.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:07.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:07.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:07.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=30130], [object_number=333, tag_tid=23, entry_tid=30129], [object_number=332, tag_tid=23, entry_tid=30128], [object_number=335, tag_tid=23, entry_tid=30127]]], state=connected] 2026-02-19T20:40:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=30130], [object_number=333, tag_tid=23, entry_tid=30129], [object_number=332, tag_tid=23, entry_tid=30128], [object_number=335, tag_tid=23, entry_tid=30127]]], state=connected] 2026-02-19T20:40:07.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=30130' 2026-02-19T20:40:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=30130' 2026-02-19T20:40:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=30130' '!=' 'object_number=333, tag_tid=23, entry_tid=29821' 2026-02-19T20:40:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=30130' = 'object_number=333, tag_tid=23, entry_tid=29821' 2026-02-19T20:40:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=30130' 2026-02-19T20:40:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:07.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=30130' 2026-02-19T20:40:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:07.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:07.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:07.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:07.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:07.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=30130' 2026-02-19T20:40:07.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:07.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:07.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30130 2026-02-19T20:40:07.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 30130 2026-02-19T20:40:07.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:07.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:07.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:07.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:07.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:07.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:07.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:07.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:07.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:07.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:07.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=30445], [object_number=332, tag_tid=23, entry_tid=30444], [object_number=335, tag_tid=23, entry_tid=30443], [object_number=334, tag_tid=23, entry_tid=30442]]], state=connected] 2026-02-19T20:40:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:07.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:07.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:07.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=30445], [object_number=332, tag_tid=23, entry_tid=30444], [object_number=335, tag_tid=23, entry_tid=30443], [object_number=334, tag_tid=23, entry_tid=30442]]], state=connected] 2026-02-19T20:40:07.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=333, tag_tid=23, entry_tid=30445' 2026-02-19T20:40:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=333, tag_tid=23, entry_tid=30445' 2026-02-19T20:40:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=30445' '!=' 'object_number=334, tag_tid=23, entry_tid=30130' 2026-02-19T20:40:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=30445' = 'object_number=334, tag_tid=23, entry_tid=30130' 2026-02-19T20:40:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=333, tag_tid=23, entry_tid=30445' 2026-02-19T20:40:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:07.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=30445' 2026-02-19T20:40:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:07.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:07.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:07.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:07.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:07.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:07.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=30445' 2026-02-19T20:40:07.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:07.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30445 2026-02-19T20:40:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 30445 2026-02-19T20:40:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=30726], [object_number=333, tag_tid=23, entry_tid=30725], [object_number=332, tag_tid=23, entry_tid=30724], [object_number=335, tag_tid=23, entry_tid=30723]]], state=connected] 2026-02-19T20:40:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:07.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:07.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:07.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:07.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:07.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:07.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:07.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:07.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:07.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=30726], [object_number=333, tag_tid=23, entry_tid=30725], [object_number=332, tag_tid=23, entry_tid=30724], [object_number=335, tag_tid=23, entry_tid=30723]]], state=connected] 2026-02-19T20:40:08.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=30726' 2026-02-19T20:40:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=30726' 2026-02-19T20:40:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=30726' '!=' 'object_number=333, tag_tid=23, entry_tid=30445' 2026-02-19T20:40:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=30726' = 'object_number=333, tag_tid=23, entry_tid=30445' 2026-02-19T20:40:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=30726' 2026-02-19T20:40:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=30726' 2026-02-19T20:40:08.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:08.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=30726' 2026-02-19T20:40:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30726 2026-02-19T20:40:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 30726 2026-02-19T20:40:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:08.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:08.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:08.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:08.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:08.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:08.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:08.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:08.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:08.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:08.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:08.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:08.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:08.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:08.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=332, tag_tid=23, entry_tid=30976], [object_number=335, tag_tid=23, entry_tid=30975], [object_number=334, tag_tid=23, entry_tid=30974], [object_number=333, tag_tid=23, entry_tid=30973]]], state=connected] 2026-02-19T20:40:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:08.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=332, tag_tid=23, entry_tid=30976], [object_number=335, tag_tid=23, entry_tid=30975], [object_number=334, tag_tid=23, entry_tid=30974], [object_number=333, tag_tid=23, entry_tid=30973]]], state=connected] 2026-02-19T20:40:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=332, tag_tid=23, entry_tid=30976' 2026-02-19T20:40:08.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=332, tag_tid=23, entry_tid=30976' 2026-02-19T20:40:08.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=332, tag_tid=23, entry_tid=30976' '!=' 'object_number=334, tag_tid=23, entry_tid=30726' 2026-02-19T20:40:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=332, tag_tid=23, entry_tid=30976' = 'object_number=334, tag_tid=23, entry_tid=30726' 2026-02-19T20:40:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=332, tag_tid=23, entry_tid=30976' 2026-02-19T20:40:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=332, tag_tid=23, entry_tid=30976' 2026-02-19T20:40:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:08.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=332, tag_tid=23, entry_tid=30976' 2026-02-19T20:40:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30976 2026-02-19T20:40:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 30976 2026-02-19T20:40:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:08.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:08.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:08.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:08.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:08.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:08.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:08.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:08.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:08.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:08.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=31249], [object_number=332, tag_tid=23, entry_tid=31248], [object_number=335, tag_tid=23, entry_tid=31247], [object_number=334, tag_tid=23, entry_tid=31246]]], state=connected] 2026-02-19T20:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:08.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:08.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:08.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:08.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=333, tag_tid=23, entry_tid=31249], [object_number=332, tag_tid=23, entry_tid=31248], [object_number=335, tag_tid=23, entry_tid=31247], [object_number=334, tag_tid=23, entry_tid=31246]]], state=connected] 2026-02-19T20:40:08.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=333, tag_tid=23, entry_tid=31249' 2026-02-19T20:40:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=333, tag_tid=23, entry_tid=31249' 2026-02-19T20:40:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=31249' '!=' 'object_number=332, tag_tid=23, entry_tid=30976' 2026-02-19T20:40:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=333, tag_tid=23, entry_tid=31249' = 'object_number=332, tag_tid=23, entry_tid=30976' 2026-02-19T20:40:08.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=333, tag_tid=23, entry_tid=31249' 2026-02-19T20:40:08.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:08.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:08.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:08.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=31249' 2026-02-19T20:40:08.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:08.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:08.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:08.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:08.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:08.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=333, tag_tid=23, entry_tid=31249' 2026-02-19T20:40:08.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:08.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31249 2026-02-19T20:40:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 31249 2026-02-19T20:40:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:09.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:09.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:09.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:09.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:09.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:09.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:09.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:09.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:09.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:09.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:09.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:09.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:09.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:09.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:09.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=31550], [object_number=333, tag_tid=23, entry_tid=31549], [object_number=332, tag_tid=23, entry_tid=31548], [object_number=335, tag_tid=23, entry_tid=31547]]], state=connected] 2026-02-19T20:40:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:09.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:09.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:09.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:09.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:09.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=31550], [object_number=333, tag_tid=23, entry_tid=31549], [object_number=332, tag_tid=23, entry_tid=31548], [object_number=335, tag_tid=23, entry_tid=31547]]], state=connected] 2026-02-19T20:40:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=31550' 2026-02-19T20:40:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=31550' 2026-02-19T20:40:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=31550' '!=' 'object_number=333, tag_tid=23, entry_tid=31249' 2026-02-19T20:40:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=31550' = 'object_number=333, tag_tid=23, entry_tid=31249' 2026-02-19T20:40:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=31550' 2026-02-19T20:40:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=31550' 2026-02-19T20:40:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:09.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=31550' 2026-02-19T20:40:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31550 2026-02-19T20:40:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 31550 2026-02-19T20:40:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:09.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:09.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:09.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:09.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:09.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:09.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:09.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:09.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:09.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:09.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:09.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:09.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:09.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:09.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=31846], [object_number=333, tag_tid=23, entry_tid=31845], [object_number=332, tag_tid=23, entry_tid=31844], [object_number=335, tag_tid=23, entry_tid=31843]]], state=connected] 2026-02-19T20:40:09.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:09.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:09.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:09.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:09.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:09.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:09.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:09.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:09.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:09.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:09.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:09.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:09.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:09.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:09.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 83 2026-02-19T20:40:09.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:09.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:09.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:09.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=334, tag_tid=23, entry_tid=31846], [object_number=333, tag_tid=23, entry_tid=31845], [object_number=332, tag_tid=23, entry_tid=31844], [object_number=335, tag_tid=23, entry_tid=31843]]], state=connected] 2026-02-19T20:40:09.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=334, tag_tid=23, entry_tid=31846' 2026-02-19T20:40:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=334, tag_tid=23, entry_tid=31846' 2026-02-19T20:40:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=31846' '!=' 'object_number=334, tag_tid=23, entry_tid=31550' 2026-02-19T20:40:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=334, tag_tid=23, entry_tid=31846' = 'object_number=334, tag_tid=23, entry_tid=31550' 2026-02-19T20:40:09.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=334, tag_tid=23, entry_tid=31846' 2026-02-19T20:40:09.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:09.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:09.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:09.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:09.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=31846' 2026-02-19T20:40:09.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:09.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:09.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=334, tag_tid=23, entry_tid=31846' 2026-02-19T20:40:09.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:09.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31846 2026-02-19T20:40:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 31846 2026-02-19T20:40:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:09.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:09.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:09.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:09.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:09.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:09.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:10.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:10.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:10.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:10.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:10.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:10.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:10.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:10.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:10.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:10.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:10.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:10.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:10.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:10.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:10.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:10.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:10.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:10.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=32110], [object_number=337, tag_tid=23, entry_tid=32109], [object_number=336, tag_tid=23, entry_tid=32108], [object_number=339, tag_tid=23, entry_tid=32107]]], state=connected] 2026-02-19T20:40:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=32110], [object_number=337, tag_tid=23, entry_tid=32109], [object_number=336, tag_tid=23, entry_tid=32108], [object_number=339, tag_tid=23, entry_tid=32107]]], state=connected] 2026-02-19T20:40:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=338, tag_tid=23, entry_tid=32110' 2026-02-19T20:40:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=338, tag_tid=23, entry_tid=32110' 2026-02-19T20:40:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=32110' '!=' 'object_number=334, tag_tid=23, entry_tid=31846' 2026-02-19T20:40:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=32110' = 'object_number=334, tag_tid=23, entry_tid=31846' 2026-02-19T20:40:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=338, tag_tid=23, entry_tid=32110' 2026-02-19T20:40:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:10.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=338, tag_tid=23, entry_tid=32110' 2026-02-19T20:40:10.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:10.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:10.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:10.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:10.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:10.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:10.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=338, tag_tid=23, entry_tid=32110' 2026-02-19T20:40:10.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:10.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32110 2026-02-19T20:40:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 32110 2026-02-19T20:40:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:10.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:10.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:10.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:10.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:10.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:10.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:10.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:10.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:10.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=339, tag_tid=23, entry_tid=32507], [object_number=338, tag_tid=23, entry_tid=32506], [object_number=337, tag_tid=23, entry_tid=32505], [object_number=336, tag_tid=23, entry_tid=32504]]], state=connected] 2026-02-19T20:40:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:10.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:10.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:10.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:11.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:11.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:11.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:11.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:11.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=339, tag_tid=23, entry_tid=32507], [object_number=338, tag_tid=23, entry_tid=32506], [object_number=337, tag_tid=23, entry_tid=32505], [object_number=336, tag_tid=23, entry_tid=32504]]], state=connected] 2026-02-19T20:40:11.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=339, tag_tid=23, entry_tid=32507' 2026-02-19T20:40:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=339, tag_tid=23, entry_tid=32507' 2026-02-19T20:40:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=339, tag_tid=23, entry_tid=32507' '!=' 'object_number=338, tag_tid=23, entry_tid=32110' 2026-02-19T20:40:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=339, tag_tid=23, entry_tid=32507' = 'object_number=338, tag_tid=23, entry_tid=32110' 2026-02-19T20:40:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=339, tag_tid=23, entry_tid=32507' 2026-02-19T20:40:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:11.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:11.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=339, tag_tid=23, entry_tid=32507' 2026-02-19T20:40:11.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:11.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:11.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:11.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=339, tag_tid=23, entry_tid=32507' 2026-02-19T20:40:11.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:11.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32507 2026-02-19T20:40:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 32507 2026-02-19T20:40:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:11.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:11.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:11.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:11.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:11.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:11.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:11.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:11.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:11.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:11.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:11.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:11.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=32934], [object_number=337, tag_tid=23, entry_tid=32933], [object_number=336, tag_tid=23, entry_tid=32932], [object_number=339, tag_tid=23, entry_tid=32931]]], state=connected] 2026-02-19T20:40:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:11.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:11.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:11.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:11.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:11.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:11.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:11.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:11.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:11.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:11.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:11.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:11.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:11.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:11.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:11.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:11.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:11.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:11.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:11.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:11.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=32934], [object_number=337, tag_tid=23, entry_tid=32933], [object_number=336, tag_tid=23, entry_tid=32932], [object_number=339, tag_tid=23, entry_tid=32931]]], state=connected] 2026-02-19T20:40:11.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=338, tag_tid=23, entry_tid=32934' 2026-02-19T20:40:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=338, tag_tid=23, entry_tid=32934' 2026-02-19T20:40:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=32934' '!=' 'object_number=339, tag_tid=23, entry_tid=32507' 2026-02-19T20:40:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=32934' = 'object_number=339, tag_tid=23, entry_tid=32507' 2026-02-19T20:40:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=338, tag_tid=23, entry_tid=32934' 2026-02-19T20:40:11.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:11.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:11.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=338, tag_tid=23, entry_tid=32934' 2026-02-19T20:40:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:11.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=338, tag_tid=23, entry_tid=32934' 2026-02-19T20:40:11.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:11.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:11.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32934 2026-02-19T20:40:11.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 32934 2026-02-19T20:40:11.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:11.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:11.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:11.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:11.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:11.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:11.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:11.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:11.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:11.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:11.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:11.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:11.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:11.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:11.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:11.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:11.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:11.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:11.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:11.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:11.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:11.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:11.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:11.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:11.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:11.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:11.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:11.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:11.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:11.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:11.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:11.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:11.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:11.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:11.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:11.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=339, tag_tid=23, entry_tid=33231], [object_number=338, tag_tid=23, entry_tid=33230], [object_number=337, tag_tid=23, entry_tid=33229], [object_number=336, tag_tid=23, entry_tid=33228]]], state=connected] 2026-02-19T20:40:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:12.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:12.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:12.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:12.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=339, tag_tid=23, entry_tid=33231], [object_number=338, tag_tid=23, entry_tid=33230], [object_number=337, tag_tid=23, entry_tid=33229], [object_number=336, tag_tid=23, entry_tid=33228]]], state=connected] 2026-02-19T20:40:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=339, tag_tid=23, entry_tid=33231' 2026-02-19T20:40:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=339, tag_tid=23, entry_tid=33231' 2026-02-19T20:40:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=339, tag_tid=23, entry_tid=33231' '!=' 'object_number=338, tag_tid=23, entry_tid=32934' 2026-02-19T20:40:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=339, tag_tid=23, entry_tid=33231' = 'object_number=338, tag_tid=23, entry_tid=32934' 2026-02-19T20:40:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=339, tag_tid=23, entry_tid=33231' 2026-02-19T20:40:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=339, tag_tid=23, entry_tid=33231' 2026-02-19T20:40:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=339, tag_tid=23, entry_tid=33231' 2026-02-19T20:40:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33231 2026-02-19T20:40:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 33231 2026-02-19T20:40:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:12.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:12.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:12.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:12.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:12.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:12.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:12.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:12.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:12.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:12.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:12.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:12.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:12.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:12.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:12.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:12.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:12.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:12.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:12.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=337, tag_tid=23, entry_tid=33609], [object_number=336, tag_tid=23, entry_tid=33608], [object_number=339, tag_tid=23, entry_tid=33607], [object_number=338, tag_tid=23, entry_tid=33606]]], state=connected] 2026-02-19T20:40:12.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:12.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=337, tag_tid=23, entry_tid=33609], [object_number=336, tag_tid=23, entry_tid=33608], [object_number=339, tag_tid=23, entry_tid=33607], [object_number=338, tag_tid=23, entry_tid=33606]]], state=connected] 2026-02-19T20:40:12.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=337, tag_tid=23, entry_tid=33609' 2026-02-19T20:40:12.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=337, tag_tid=23, entry_tid=33609' 2026-02-19T20:40:12.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=337, tag_tid=23, entry_tid=33609' '!=' 'object_number=339, tag_tid=23, entry_tid=33231' 2026-02-19T20:40:12.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:12.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=337, tag_tid=23, entry_tid=33609' = 'object_number=339, tag_tid=23, entry_tid=33231' 2026-02-19T20:40:12.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=337, tag_tid=23, entry_tid=33609' 2026-02-19T20:40:12.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:12.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:12.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=23, entry_tid=33609' 2026-02-19T20:40:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:12.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=23, entry_tid=33609' 2026-02-19T20:40:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:12.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33609 2026-02-19T20:40:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 33609 2026-02-19T20:40:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:12.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:12.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:12.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:12.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:12.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:12.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:12.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:12.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:12.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:12.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:12.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:12.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:12.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:12.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:12.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:12.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:12.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:12.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:12.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:12.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:12.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:12.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:12.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:12.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:12.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=33666], [object_number=337, tag_tid=23, entry_tid=33665], [object_number=336, tag_tid=23, entry_tid=33664], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=33666], [object_number=337, tag_tid=23, entry_tid=33665], [object_number=336, tag_tid=23, entry_tid=33664], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:13.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=33666' '!=' 'object_number=337, tag_tid=23, entry_tid=33609' 2026-02-19T20:40:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:40:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=33666' = 'object_number=337, tag_tid=23, entry_tid=33609' 2026-02-19T20:40:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=24 2026-02-19T20:40:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:40:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=23 2026-02-19T20:40:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:40:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:40:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:40:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=33666 2026-02-19T20:40:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 24 = 23 -a 1 -le 33666 2026-02-19T20:40:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:40:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:13.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:13.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:13.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:13.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:13.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:13.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:13.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=33666], [object_number=337, tag_tid=23, entry_tid=33665], [object_number=336, tag_tid=23, entry_tid=33664], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:13.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=33666], [object_number=337, tag_tid=23, entry_tid=33665], [object_number=336, tag_tid=23, entry_tid=33664], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:13.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=33666' '!=' 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:40:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:14.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:14.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:14.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:14.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:14.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:14.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:14.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=33666], [object_number=337, tag_tid=23, entry_tid=33665], [object_number=336, tag_tid=23, entry_tid=33664], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:14.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=33666], [object_number=337, tag_tid=23, entry_tid=33665], [object_number=336, tag_tid=23, entry_tid=33664], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=33666' '!=' 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:40:15.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:15.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:15.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:15.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:15.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:15.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:15.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:15.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:15.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:15.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:15.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=33666], [object_number=337, tag_tid=23, entry_tid=33665], [object_number=336, tag_tid=23, entry_tid=33664], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:15.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:15.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=33666], [object_number=337, tag_tid=23, entry_tid=33665], [object_number=336, tag_tid=23, entry_tid=33664], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=33666' '!=' 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:17.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:17.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:17.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:17.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:17.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:17.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:17.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=33666], [object_number=337, tag_tid=23, entry_tid=33665], [object_number=336, tag_tid=23, entry_tid=33664], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=338, tag_tid=23, entry_tid=33666], [object_number=337, tag_tid=23, entry_tid=33665], [object_number=336, tag_tid=23, entry_tid=33664], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=338, tag_tid=23, entry_tid=33666' '!=' 'object_number=338, tag_tid=23, entry_tid=33666' 2026-02-19T20:40:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:40:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:40:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:40:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:40:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:40:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:40:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:40:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:40:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:40:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:40:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:40:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:40:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:40:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:40:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:40:19.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:40:19.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:40:19.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:40:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:40:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:40:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:40:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:40:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:40:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:19.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:19.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:40:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:40:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:40:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:40:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:40:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:40:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:40:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:40:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:40:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:40:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 84 2026-02-19T20:40:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 84 2026-02-19T20:40:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:40:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=337, tag_tid=24, entry_tid=1], [object_number=336, tag_tid=24, entry_tid=0], [object_number=338, tag_tid=23, entry_tid=33666], [object_number=339, tag_tid=23, entry_tid=33663]]], state=connected] 2026-02-19T20:40:19.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:40:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=337, tag_tid=24, entry_tid=1' -a 'object_number=337, tag_tid=24, entry_tid=1' = 'object_number=337, tag_tid=24, entry_tid=1' 2026-02-19T20:40:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2026-02-19T20:40:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap8 2026-02-19T20:40:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:40:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:40:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:40:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2026-02-19T20:40:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 15 2026-02-19T20:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:40:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:39:10 2026 2026-02-19T20:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:39:41 2026 2026-02-19T20:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:40:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:40:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n snap7-clean ']' 2026-02-19T20:40:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-19T20:40:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap7-clean 2026-02-19T20:40:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:40:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:40:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:40:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:40:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:40:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:40:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:40:21.509 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:40:21.503+0000 7f939a72f640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-19T20:40:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:40:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:40:22.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:40:22.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:22.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap8 2026-02-19T20:40:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap8 2026-02-19T20:40:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:40:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:40:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:40:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:40:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:40:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:40:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:40:23.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:40:23.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:40:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:40:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name=snap8-clean 2026-02-19T20:40:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap8-clean 2026-02-19T20:40:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:40:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap8-clean 2026-02-19T20:40:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2026-02-19T20:40:24.132 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T20:40:24.132 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T20:40:24.133 INFO:tasks.rbd_mirror_thrash:waiting for 1.2192108887169928 secs before reviving daemons 2026-02-19T20:40:24.134 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T20:40:24.125+0000 7f67b0939640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 51286) UID: 1000 2026-02-19T20:40:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:40:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror test 2026-02-19T20:40:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:40:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:40:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=9 2026-02-19T20:40:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:40:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-19T20:40:25.353 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T20:40:25.353 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:40:25.354 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T20:40:25.354 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T20:40:25.354 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T20:40:25.356 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T20:40:25.357 INFO:tasks.rbd_mirror_thrash:waiting for 115.2122866991825 secs before thrashing 2026-02-19T20:40:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 54916096 bytes 2026-02-19T20:40:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 79488000 bytes 2026-02-19T20:40:26.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 139716608 bytes 2026-02-19T20:40:27.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 172262400 bytes 2026-02-19T20:40:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 227138560 bytes 2026-02-19T20:40:28.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 240707584 bytes 2026-02-19T20:40:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 321049600 bytes 2026-02-19T20:40:30.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 355044352 bytes 2026-02-19T20:40:31.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 385182720 bytes 2026-02-19T20:40:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 441199616 bytes 2026-02-19T20:40:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 445809664 bytes 2026-02-19T20:40:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 471237632 bytes 2026-02-19T20:40:33.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 502810624 bytes 2026-02-19T20:40:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:40:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:40:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:40:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-19T20:40:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:40:33.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:40:33.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:40:33.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:40:33.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:40:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-19T20:40:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:40:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:40:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:40:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:40:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:40:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='test: 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: global_id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+replaying 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: replaying, {"bytes_per_second":8768022.4,"entries_behind_primary":1,"entries_per_second":550.0,"non_primary_position":{"entry_tid":33666,"object_number":338,"tag_tid":23},"primary_position":{"entry_tid":1,"object_number":337,"tag_tid":24},"seconds_until_synced":0} 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: service: mirror.1 on trial151 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:40:17 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: peer_sites: 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: name: cluster2 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+stopped 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: local image is primary 2026-02-19T20:40:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:40:18' 2026-02-19T20:40:34.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ state: .*up+replaying' 2026-02-19T20:40:34.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: state: up+replaying 2026-02-19T20:40:34.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ description: .*' 2026-02-19T20:40:34.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: description: replaying, {"bytes_per_second":8768022.4,"entries_behind_primary":1,"entries_per_second":550.0,"non_primary_position":{"entry_tid":33666,"object_number":338,"tag_tid":23},"primary_position":{"entry_tid":1,"object_number":337,"tag_tid":24},"seconds_until_synced":0} 2026-02-19T20:40:34.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:40:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:40:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^up+' 2026-02-19T20:40:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:40:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep 'service: *mirror..* on ' 2026-02-19T20:40:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: service: mirror.1 on trial151 2026-02-19T20:40:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update 2026-02-19T20:40:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-19T20:40:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:40:17' 2026-02-19T20:40:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-19 20:40:17' 2026-02-19T20:40:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:40:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:40:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'prev_last_update=2026-02-19 20:40:17' 2026-02-19T20:40:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:40:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:40:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-19T20:40:34.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='OKOKOK142064262mirror.1trial15120.3.0-5336-g802bf111falseOK42104271mirror.2trial15120.3.0-5336-g802bf111trueOK43714374mirror.3trial15120.3.0-5336-g802bf111falseOK44734476mirror.0trial15120.3.0-5336-g802bf111falseOKtest959deb13-de1a-48d9-a341-1f95416f8379up+replayingreplaying, {"bytes_per_second":8768022.4,"entries_behind_primary":1,"entries_per_second":550.0,"non_primary_position":{"entry_tid":33666,"object_number":338,"tag_tid":23},"primary_position":{"entry_tid":1,"object_number":337,"tag_tid":24},"seconds_until_synced":0}42064262mirror.1trial1512026-02-19 20:40:17cluster2a054b997-3cfb-48e6-978a-70d6a85e56deup+stoppedlocal image is primary2026-02-19 20:40:18' 2026-02-19T20:40:34.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update state 2026-02-19T20:40:34.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-19T20:40:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:40:17' 2026-02-19T20:40:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-19T20:40:34.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=up+replaying 2026-02-19T20:40:34.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:40:34.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep up+replaying 2026-02-19T20:40:34.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:40:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:40:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap9 2026-02-19T20:40:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap9 2026-02-19T20:40:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:40:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:40:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:40:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap9 2026-02-19T20:40:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:41:06.657 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8164830901157952) > weight (0.25) 2026-02-19T20:41:06.657 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7750573806396299) > weight (0.25) 2026-02-19T20:41:06.657 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5392563655364777) > weight (0.25) 2026-02-19T20:41:06.657 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.3726763450347771) > weight (0.25) 2026-02-19T20:41:06.657 INFO:tasks.rbd_mirror_thrash:waiting for 83.38305960245575 secs before thrashing 2026-02-19T20:41:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:41:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:41:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:41:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:41:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:41:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:41:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:41:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:41:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:41:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:41:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:41:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:41:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:41:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:41:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:41:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:41:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:41:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:41:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:41:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:41:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:07.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:41:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:41:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:07.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:07.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:07.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:41:07.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:41:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:41:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:41:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:41:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:41:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:41:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:41:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:41:07.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:41:07.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:41:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:41:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:41:07.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:41:07.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:41:07.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:41:07.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:07.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:41:07.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:41:07.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:41:07.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:41:07.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:08.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:08.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:08.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:08.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:08.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=24729], [object_number=360, tag_tid=26, entry_tid=24728], [object_number=363, tag_tid=26, entry_tid=24727], [object_number=362, tag_tid=26, entry_tid=24726]]], state=connected] 2026-02-19T20:41:08.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=24729], [object_number=360, tag_tid=26, entry_tid=24728], [object_number=363, tag_tid=26, entry_tid=24727], [object_number=362, tag_tid=26, entry_tid=24726]]], state=connected] 2026-02-19T20:41:08.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=24729' 2026-02-19T20:41:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=24729' 2026-02-19T20:41:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=24729' '!=' '' 2026-02-19T20:41:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=24729' = '' 2026-02-19T20:41:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=24729' 2026-02-19T20:41:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:08.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:08.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:08.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:08.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=24729' 2026-02-19T20:41:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:08.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=24729' 2026-02-19T20:41:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24729 2026-02-19T20:41:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 24729 2026-02-19T20:41:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:08.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:08.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:08.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:08.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:08.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:08.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:08.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:08.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:08.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:08.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:08.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:08.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:08.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:08.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:08.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:08.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:08.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:08.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:08.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:08.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:08.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:08.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=24980], [object_number=363, tag_tid=26, entry_tid=24979], [object_number=362, tag_tid=26, entry_tid=24978], [object_number=361, tag_tid=26, entry_tid=24977]]], state=connected] 2026-02-19T20:41:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:08.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:08.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:08.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:08.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:08.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=24980], [object_number=363, tag_tid=26, entry_tid=24979], [object_number=362, tag_tid=26, entry_tid=24978], [object_number=361, tag_tid=26, entry_tid=24977]]], state=connected] 2026-02-19T20:41:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=360, tag_tid=26, entry_tid=24980' 2026-02-19T20:41:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=360, tag_tid=26, entry_tid=24980' 2026-02-19T20:41:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=24980' '!=' 'object_number=361, tag_tid=26, entry_tid=24729' 2026-02-19T20:41:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=24980' = 'object_number=361, tag_tid=26, entry_tid=24729' 2026-02-19T20:41:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=360, tag_tid=26, entry_tid=24980' 2026-02-19T20:41:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=24980' 2026-02-19T20:41:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:08.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=24980' 2026-02-19T20:41:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:08.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24980 2026-02-19T20:41:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 24980 2026-02-19T20:41:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:08.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:09.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:09.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:09.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:09.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:09.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:09.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:09.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=25226], [object_number=361, tag_tid=26, entry_tid=25225], [object_number=360, tag_tid=26, entry_tid=25224], [object_number=363, tag_tid=26, entry_tid=25223]]], state=connected] 2026-02-19T20:41:09.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:09.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:09.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:09.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:09.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=25226], [object_number=361, tag_tid=26, entry_tid=25225], [object_number=360, tag_tid=26, entry_tid=25224], [object_number=363, tag_tid=26, entry_tid=25223]]], state=connected] 2026-02-19T20:41:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=362, tag_tid=26, entry_tid=25226' 2026-02-19T20:41:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=362, tag_tid=26, entry_tid=25226' 2026-02-19T20:41:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=25226' '!=' 'object_number=360, tag_tid=26, entry_tid=24980' 2026-02-19T20:41:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=25226' = 'object_number=360, tag_tid=26, entry_tid=24980' 2026-02-19T20:41:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=362, tag_tid=26, entry_tid=25226' 2026-02-19T20:41:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=25226' 2026-02-19T20:41:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:09.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:09.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=25226' 2026-02-19T20:41:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25226 2026-02-19T20:41:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 25226 2026-02-19T20:41:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:09.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:09.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:09.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:09.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:09.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:09.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:09.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:09.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=363, tag_tid=26, entry_tid=25487], [object_number=362, tag_tid=26, entry_tid=25486], [object_number=361, tag_tid=26, entry_tid=25485], [object_number=360, tag_tid=26, entry_tid=25484]]], state=connected] 2026-02-19T20:41:09.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:09.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:09.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:09.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:09.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=363, tag_tid=26, entry_tid=25487], [object_number=362, tag_tid=26, entry_tid=25486], [object_number=361, tag_tid=26, entry_tid=25485], [object_number=360, tag_tid=26, entry_tid=25484]]], state=connected] 2026-02-19T20:41:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=363, tag_tid=26, entry_tid=25487' 2026-02-19T20:41:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=363, tag_tid=26, entry_tid=25487' 2026-02-19T20:41:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=363, tag_tid=26, entry_tid=25487' '!=' 'object_number=362, tag_tid=26, entry_tid=25226' 2026-02-19T20:41:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=363, tag_tid=26, entry_tid=25487' = 'object_number=362, tag_tid=26, entry_tid=25226' 2026-02-19T20:41:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=363, tag_tid=26, entry_tid=25487' 2026-02-19T20:41:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=363, tag_tid=26, entry_tid=25487' 2026-02-19T20:41:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=363, tag_tid=26, entry_tid=25487' 2026-02-19T20:41:09.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:09.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:09.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25487 2026-02-19T20:41:09.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 25487 2026-02-19T20:41:09.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:09.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:09.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:09.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:09.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:09.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:09.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:09.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:09.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:09.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:09.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:09.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:09.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:09.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:09.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:09.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:09.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:09.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:09.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:09.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:09.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:09.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:09.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:09.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:09.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:09.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:09.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:09.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:09.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=363, tag_tid=26, entry_tid=25771], [object_number=362, tag_tid=26, entry_tid=25770], [object_number=361, tag_tid=26, entry_tid=25769], [object_number=360, tag_tid=26, entry_tid=25768]]], state=connected] 2026-02-19T20:41:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=363, tag_tid=26, entry_tid=25771], [object_number=362, tag_tid=26, entry_tid=25770], [object_number=361, tag_tid=26, entry_tid=25769], [object_number=360, tag_tid=26, entry_tid=25768]]], state=connected] 2026-02-19T20:41:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=363, tag_tid=26, entry_tid=25771' 2026-02-19T20:41:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=363, tag_tid=26, entry_tid=25771' 2026-02-19T20:41:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=363, tag_tid=26, entry_tid=25771' '!=' 'object_number=363, tag_tid=26, entry_tid=25487' 2026-02-19T20:41:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=363, tag_tid=26, entry_tid=25771' = 'object_number=363, tag_tid=26, entry_tid=25487' 2026-02-19T20:41:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=363, tag_tid=26, entry_tid=25771' 2026-02-19T20:41:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=363, tag_tid=26, entry_tid=25771' 2026-02-19T20:41:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:10.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:10.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:10.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=363, tag_tid=26, entry_tid=25771' 2026-02-19T20:41:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25771 2026-02-19T20:41:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 25771 2026-02-19T20:41:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:10.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:10.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:10.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:10.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=26040], [object_number=363, tag_tid=26, entry_tid=26039], [object_number=362, tag_tid=26, entry_tid=26038], [object_number=361, tag_tid=26, entry_tid=26037]]], state=connected] 2026-02-19T20:41:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=26040], [object_number=363, tag_tid=26, entry_tid=26039], [object_number=362, tag_tid=26, entry_tid=26038], [object_number=361, tag_tid=26, entry_tid=26037]]], state=connected] 2026-02-19T20:41:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=360, tag_tid=26, entry_tid=26040' 2026-02-19T20:41:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=360, tag_tid=26, entry_tid=26040' 2026-02-19T20:41:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=26040' '!=' 'object_number=363, tag_tid=26, entry_tid=25771' 2026-02-19T20:41:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=26040' = 'object_number=363, tag_tid=26, entry_tid=25771' 2026-02-19T20:41:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=360, tag_tid=26, entry_tid=26040' 2026-02-19T20:41:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:10.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:10.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=26040' 2026-02-19T20:41:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=26040' 2026-02-19T20:41:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26040 2026-02-19T20:41:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 26040 2026-02-19T20:41:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:10.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:10.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:10.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:10.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:10.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:10.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:10.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:10.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:10.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:10.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:10.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:10.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:10.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:10.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:10.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:10.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=26300], [object_number=363, tag_tid=26, entry_tid=26299], [object_number=362, tag_tid=26, entry_tid=26298], [object_number=361, tag_tid=26, entry_tid=26297]]], state=connected] 2026-02-19T20:41:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:10.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:10.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:10.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=26300], [object_number=363, tag_tid=26, entry_tid=26299], [object_number=362, tag_tid=26, entry_tid=26298], [object_number=361, tag_tid=26, entry_tid=26297]]], state=connected] 2026-02-19T20:41:10.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:10.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=360, tag_tid=26, entry_tid=26300' 2026-02-19T20:41:10.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=360, tag_tid=26, entry_tid=26300' 2026-02-19T20:41:10.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=26300' '!=' 'object_number=360, tag_tid=26, entry_tid=26040' 2026-02-19T20:41:10.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:10.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=26300' = 'object_number=360, tag_tid=26, entry_tid=26040' 2026-02-19T20:41:10.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=360, tag_tid=26, entry_tid=26300' 2026-02-19T20:41:10.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:10.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:10.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:10.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=26300' 2026-02-19T20:41:10.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:10.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:10.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=26300' 2026-02-19T20:41:10.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:10.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:10.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26300 2026-02-19T20:41:10.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 26300 2026-02-19T20:41:10.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:10.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:10.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:11.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:11.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:11.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:11.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:11.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:11.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:11.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:11.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:11.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:11.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:11.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:11.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:11.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:11.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:11.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:11.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:11.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:11.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:11.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:11.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=26573], [object_number=360, tag_tid=26, entry_tid=26572], [object_number=363, tag_tid=26, entry_tid=26571], [object_number=362, tag_tid=26, entry_tid=26570]]], state=connected] 2026-02-19T20:41:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:11.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:11.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:11.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:11.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:11.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=26573], [object_number=360, tag_tid=26, entry_tid=26572], [object_number=363, tag_tid=26, entry_tid=26571], [object_number=362, tag_tid=26, entry_tid=26570]]], state=connected] 2026-02-19T20:41:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=26573' 2026-02-19T20:41:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=26573' 2026-02-19T20:41:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=26573' '!=' 'object_number=360, tag_tid=26, entry_tid=26300' 2026-02-19T20:41:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=26573' = 'object_number=360, tag_tid=26, entry_tid=26300' 2026-02-19T20:41:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=26573' 2026-02-19T20:41:11.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:11.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:11.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=26573' 2026-02-19T20:41:11.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:11.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:11.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:11.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:11.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:11.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:11.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:11.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=26573' 2026-02-19T20:41:11.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:11.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:11.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26573 2026-02-19T20:41:11.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 26573 2026-02-19T20:41:11.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:11.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:11.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:11.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:11.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:11.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:11.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:11.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:11.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:11.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:11.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:11.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:11.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:11.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:11.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:11.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=363, tag_tid=26, entry_tid=26879], [object_number=362, tag_tid=26, entry_tid=26878], [object_number=361, tag_tid=26, entry_tid=26877], [object_number=360, tag_tid=26, entry_tid=26876]]], state=connected] 2026-02-19T20:41:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:11.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:11.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:11.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:11.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:11.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=363, tag_tid=26, entry_tid=26879], [object_number=362, tag_tid=26, entry_tid=26878], [object_number=361, tag_tid=26, entry_tid=26877], [object_number=360, tag_tid=26, entry_tid=26876]]], state=connected] 2026-02-19T20:41:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=363, tag_tid=26, entry_tid=26879' 2026-02-19T20:41:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=363, tag_tid=26, entry_tid=26879' 2026-02-19T20:41:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=363, tag_tid=26, entry_tid=26879' '!=' 'object_number=361, tag_tid=26, entry_tid=26573' 2026-02-19T20:41:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=363, tag_tid=26, entry_tid=26879' = 'object_number=361, tag_tid=26, entry_tid=26573' 2026-02-19T20:41:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=363, tag_tid=26, entry_tid=26879' 2026-02-19T20:41:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=363, tag_tid=26, entry_tid=26879' 2026-02-19T20:41:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=363, tag_tid=26, entry_tid=26879' 2026-02-19T20:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26879 2026-02-19T20:41:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 26879 2026-02-19T20:41:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:11.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:12.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:12.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:12.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:12.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=27161], [object_number=360, tag_tid=26, entry_tid=27160], [object_number=363, tag_tid=26, entry_tid=27159], [object_number=362, tag_tid=26, entry_tid=27158]]], state=connected] 2026-02-19T20:41:12.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:12.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:12.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=27161], [object_number=360, tag_tid=26, entry_tid=27160], [object_number=363, tag_tid=26, entry_tid=27159], [object_number=362, tag_tid=26, entry_tid=27158]]], state=connected] 2026-02-19T20:41:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=27161' 2026-02-19T20:41:12.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=27161' 2026-02-19T20:41:12.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=27161' '!=' 'object_number=363, tag_tid=26, entry_tid=26879' 2026-02-19T20:41:12.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:12.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=27161' = 'object_number=363, tag_tid=26, entry_tid=26879' 2026-02-19T20:41:12.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=27161' 2026-02-19T20:41:12.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=27161' 2026-02-19T20:41:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:12.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=27161' 2026-02-19T20:41:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27161 2026-02-19T20:41:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 27161 2026-02-19T20:41:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:12.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:12.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:12.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:12.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:12.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:12.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:12.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:12.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:12.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:12.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:12.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:12.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:12.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:12.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=27377], [object_number=364, tag_tid=26, entry_tid=27376], [object_number=367, tag_tid=26, entry_tid=27375], [object_number=362, tag_tid=26, entry_tid=27374]]], state=connected] 2026-02-19T20:41:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:12.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:12.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:12.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:12.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 90 2026-02-19T20:41:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=27377], [object_number=364, tag_tid=26, entry_tid=27376], [object_number=367, tag_tid=26, entry_tid=27375], [object_number=362, tag_tid=26, entry_tid=27374]]], state=connected] 2026-02-19T20:41:12.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=27377' 2026-02-19T20:41:12.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=27377' 2026-02-19T20:41:12.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=27377' '!=' 'object_number=361, tag_tid=26, entry_tid=27161' 2026-02-19T20:41:12.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:12.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=27377' = 'object_number=361, tag_tid=26, entry_tid=27161' 2026-02-19T20:41:12.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=27377' 2026-02-19T20:41:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:12.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=27377' 2026-02-19T20:41:12.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:12.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:12.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:12.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:12.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:12.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:12.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:12.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=27377' 2026-02-19T20:41:12.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:12.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:12.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27377 2026-02-19T20:41:12.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 27377 2026-02-19T20:41:12.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:12.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:12.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:12.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:12.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:12.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:12.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:12.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:12.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:12.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:12.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:12.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:12.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:12.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:12.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=27643], [object_number=366, tag_tid=26, entry_tid=27642], [object_number=365, tag_tid=26, entry_tid=27641], [object_number=364, tag_tid=26, entry_tid=27640]]], state=connected] 2026-02-19T20:41:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:13.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:13.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:13.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:13.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:13.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=27643], [object_number=366, tag_tid=26, entry_tid=27642], [object_number=365, tag_tid=26, entry_tid=27641], [object_number=364, tag_tid=26, entry_tid=27640]]], state=connected] 2026-02-19T20:41:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=27643' 2026-02-19T20:41:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=27643' 2026-02-19T20:41:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=27643' '!=' 'object_number=365, tag_tid=26, entry_tid=27377' 2026-02-19T20:41:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=27643' = 'object_number=365, tag_tid=26, entry_tid=27377' 2026-02-19T20:41:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=27643' 2026-02-19T20:41:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=27643' 2026-02-19T20:41:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=27643' 2026-02-19T20:41:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27643 2026-02-19T20:41:13.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 27643 2026-02-19T20:41:13.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:13.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:13.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:13.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:13.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:13.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:13.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:13.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:13.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:13.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=27932], [object_number=367, tag_tid=26, entry_tid=27931], [object_number=366, tag_tid=26, entry_tid=27930], [object_number=365, tag_tid=26, entry_tid=27929]]], state=connected] 2026-02-19T20:41:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:13.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:13.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:13.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:13.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:13.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:13.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=27932], [object_number=367, tag_tid=26, entry_tid=27931], [object_number=366, tag_tid=26, entry_tid=27930], [object_number=365, tag_tid=26, entry_tid=27929]]], state=connected] 2026-02-19T20:41:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=364, tag_tid=26, entry_tid=27932' 2026-02-19T20:41:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=364, tag_tid=26, entry_tid=27932' 2026-02-19T20:41:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=27932' '!=' 'object_number=367, tag_tid=26, entry_tid=27643' 2026-02-19T20:41:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=27932' = 'object_number=367, tag_tid=26, entry_tid=27643' 2026-02-19T20:41:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=364, tag_tid=26, entry_tid=27932' 2026-02-19T20:41:13.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:13.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:13.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=27932' 2026-02-19T20:41:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=27932' 2026-02-19T20:41:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=27932 2026-02-19T20:41:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 27932 2026-02-19T20:41:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:13.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:13.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:13.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:13.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:13.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:13.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:13.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:13.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:13.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:13.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:13.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:13.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:13.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:13.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:13.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:13.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:13.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:13.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:13.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=28207], [object_number=366, tag_tid=26, entry_tid=28206], [object_number=365, tag_tid=26, entry_tid=28205], [object_number=364, tag_tid=26, entry_tid=28204]]], state=connected] 2026-02-19T20:41:13.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=28207], [object_number=366, tag_tid=26, entry_tid=28206], [object_number=365, tag_tid=26, entry_tid=28205], [object_number=364, tag_tid=26, entry_tid=28204]]], state=connected] 2026-02-19T20:41:13.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=28207' 2026-02-19T20:41:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=28207' 2026-02-19T20:41:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=28207' '!=' 'object_number=364, tag_tid=26, entry_tid=27932' 2026-02-19T20:41:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=28207' = 'object_number=364, tag_tid=26, entry_tid=27932' 2026-02-19T20:41:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=28207' 2026-02-19T20:41:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=28207' 2026-02-19T20:41:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=28207' 2026-02-19T20:41:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28207 2026-02-19T20:41:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 28207 2026-02-19T20:41:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:14.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:14.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:14.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:14.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:14.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:14.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:14.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=28510], [object_number=365, tag_tid=26, entry_tid=28509], [object_number=364, tag_tid=26, entry_tid=28508], [object_number=367, tag_tid=26, entry_tid=28507]]], state=connected] 2026-02-19T20:41:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=28510], [object_number=365, tag_tid=26, entry_tid=28509], [object_number=364, tag_tid=26, entry_tid=28508], [object_number=367, tag_tid=26, entry_tid=28507]]], state=connected] 2026-02-19T20:41:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=366, tag_tid=26, entry_tid=28510' 2026-02-19T20:41:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=366, tag_tid=26, entry_tid=28510' 2026-02-19T20:41:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=28510' '!=' 'object_number=367, tag_tid=26, entry_tid=28207' 2026-02-19T20:41:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=28510' = 'object_number=367, tag_tid=26, entry_tid=28207' 2026-02-19T20:41:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=366, tag_tid=26, entry_tid=28510' 2026-02-19T20:41:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=28510' 2026-02-19T20:41:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:14.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=28510' 2026-02-19T20:41:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28510 2026-02-19T20:41:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 28510 2026-02-19T20:41:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:14.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:14.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:14.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:14.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:14.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:14.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:14.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:14.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:14.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:14.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:14.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:14.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=28779], [object_number=366, tag_tid=26, entry_tid=28778], [object_number=365, tag_tid=26, entry_tid=28777], [object_number=364, tag_tid=26, entry_tid=28776]]], state=connected] 2026-02-19T20:41:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=28779], [object_number=366, tag_tid=26, entry_tid=28778], [object_number=365, tag_tid=26, entry_tid=28777], [object_number=364, tag_tid=26, entry_tid=28776]]], state=connected] 2026-02-19T20:41:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=28779' 2026-02-19T20:41:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=28779' 2026-02-19T20:41:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=28779' '!=' 'object_number=366, tag_tid=26, entry_tid=28510' 2026-02-19T20:41:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=28779' = 'object_number=366, tag_tid=26, entry_tid=28510' 2026-02-19T20:41:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=28779' 2026-02-19T20:41:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=28779' 2026-02-19T20:41:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:14.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=28779' 2026-02-19T20:41:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=28779 2026-02-19T20:41:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 28779 2026-02-19T20:41:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:15.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:15.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:15.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:15.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:15.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:15.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:15.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:15.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:15.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:15.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:15.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=29049], [object_number=364, tag_tid=26, entry_tid=29048], [object_number=367, tag_tid=26, entry_tid=29047], [object_number=366, tag_tid=26, entry_tid=29046]]], state=connected] 2026-02-19T20:41:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=29049], [object_number=364, tag_tid=26, entry_tid=29048], [object_number=367, tag_tid=26, entry_tid=29047], [object_number=366, tag_tid=26, entry_tid=29046]]], state=connected] 2026-02-19T20:41:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=29049' 2026-02-19T20:41:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=29049' 2026-02-19T20:41:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=29049' '!=' 'object_number=367, tag_tid=26, entry_tid=28779' 2026-02-19T20:41:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=29049' = 'object_number=367, tag_tid=26, entry_tid=28779' 2026-02-19T20:41:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=29049' 2026-02-19T20:41:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:15.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=29049' 2026-02-19T20:41:15.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:15.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=29049' 2026-02-19T20:41:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29049 2026-02-19T20:41:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 29049 2026-02-19T20:41:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:15.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:15.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:15.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:15.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:15.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:15.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:15.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:15.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:15.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:15.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=29358], [object_number=365, tag_tid=26, entry_tid=29357], [object_number=364, tag_tid=26, entry_tid=29356], [object_number=367, tag_tid=26, entry_tid=29355]]], state=connected] 2026-02-19T20:41:15.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:15.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:15.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:15.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:15.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:15.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=29358], [object_number=365, tag_tid=26, entry_tid=29357], [object_number=364, tag_tid=26, entry_tid=29356], [object_number=367, tag_tid=26, entry_tid=29355]]], state=connected] 2026-02-19T20:41:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=366, tag_tid=26, entry_tid=29358' 2026-02-19T20:41:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=366, tag_tid=26, entry_tid=29358' 2026-02-19T20:41:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=29358' '!=' 'object_number=365, tag_tid=26, entry_tid=29049' 2026-02-19T20:41:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=29358' = 'object_number=365, tag_tid=26, entry_tid=29049' 2026-02-19T20:41:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=366, tag_tid=26, entry_tid=29358' 2026-02-19T20:41:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:15.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=29358' 2026-02-19T20:41:15.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:15.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=29358' 2026-02-19T20:41:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:15.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:15.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29358 2026-02-19T20:41:15.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 29358 2026-02-19T20:41:15.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:15.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:15.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:15.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:15.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:15.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:15.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:15.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:15.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:15.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:15.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:15.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:15.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:15.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:15.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:16.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=29627], [object_number=366, tag_tid=26, entry_tid=29626], [object_number=365, tag_tid=26, entry_tid=29625], [object_number=364, tag_tid=26, entry_tid=29624]]], state=connected] 2026-02-19T20:41:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:16.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:16.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:16.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:16.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=29627], [object_number=366, tag_tid=26, entry_tid=29626], [object_number=365, tag_tid=26, entry_tid=29625], [object_number=364, tag_tid=26, entry_tid=29624]]], state=connected] 2026-02-19T20:41:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=29627' 2026-02-19T20:41:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=29627' 2026-02-19T20:41:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=29627' '!=' 'object_number=366, tag_tid=26, entry_tid=29358' 2026-02-19T20:41:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=29627' = 'object_number=366, tag_tid=26, entry_tid=29358' 2026-02-19T20:41:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=29627' 2026-02-19T20:41:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:16.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:16.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=29627' 2026-02-19T20:41:16.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:16.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=29627' 2026-02-19T20:41:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29627 2026-02-19T20:41:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 29627 2026-02-19T20:41:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:16.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:16.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:16.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:16.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:16.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=29893], [object_number=364, tag_tid=26, entry_tid=29892], [object_number=367, tag_tid=26, entry_tid=29891], [object_number=366, tag_tid=26, entry_tid=29890]]], state=connected] 2026-02-19T20:41:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:16.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:16.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:16.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:16.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:16.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=29893], [object_number=364, tag_tid=26, entry_tid=29892], [object_number=367, tag_tid=26, entry_tid=29891], [object_number=366, tag_tid=26, entry_tid=29890]]], state=connected] 2026-02-19T20:41:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=29893' 2026-02-19T20:41:16.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=29893' 2026-02-19T20:41:16.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=29893' '!=' 'object_number=367, tag_tid=26, entry_tid=29627' 2026-02-19T20:41:16.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:16.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=29893' = 'object_number=367, tag_tid=26, entry_tid=29627' 2026-02-19T20:41:16.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=29893' 2026-02-19T20:41:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=29893' 2026-02-19T20:41:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:16.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=29893' 2026-02-19T20:41:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=29893 2026-02-19T20:41:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 29893 2026-02-19T20:41:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:16.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:16.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:16.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:16.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:16.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:16.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:16.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:16.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:16.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:16.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:16.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:16.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=30174], [object_number=365, tag_tid=26, entry_tid=30173], [object_number=364, tag_tid=26, entry_tid=30172], [object_number=367, tag_tid=26, entry_tid=30171]]], state=connected] 2026-02-19T20:41:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=30174], [object_number=365, tag_tid=26, entry_tid=30173], [object_number=364, tag_tid=26, entry_tid=30172], [object_number=367, tag_tid=26, entry_tid=30171]]], state=connected] 2026-02-19T20:41:16.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=366, tag_tid=26, entry_tid=30174' 2026-02-19T20:41:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=366, tag_tid=26, entry_tid=30174' 2026-02-19T20:41:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=30174' '!=' 'object_number=365, tag_tid=26, entry_tid=29893' 2026-02-19T20:41:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=30174' = 'object_number=365, tag_tid=26, entry_tid=29893' 2026-02-19T20:41:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=366, tag_tid=26, entry_tid=30174' 2026-02-19T20:41:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:16.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=30174' 2026-02-19T20:41:16.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:16.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=30174' 2026-02-19T20:41:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30174 2026-02-19T20:41:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 30174 2026-02-19T20:41:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:17.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:17.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:17.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:17.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:17.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=30464], [object_number=367, tag_tid=26, entry_tid=30463], [object_number=366, tag_tid=26, entry_tid=30462], [object_number=365, tag_tid=26, entry_tid=30461]]], state=connected] 2026-02-19T20:41:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=30464], [object_number=367, tag_tid=26, entry_tid=30463], [object_number=366, tag_tid=26, entry_tid=30462], [object_number=365, tag_tid=26, entry_tid=30461]]], state=connected] 2026-02-19T20:41:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=364, tag_tid=26, entry_tid=30464' 2026-02-19T20:41:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=364, tag_tid=26, entry_tid=30464' 2026-02-19T20:41:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=30464' '!=' 'object_number=366, tag_tid=26, entry_tid=30174' 2026-02-19T20:41:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=30464' = 'object_number=366, tag_tid=26, entry_tid=30174' 2026-02-19T20:41:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=364, tag_tid=26, entry_tid=30464' 2026-02-19T20:41:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:17.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=30464' 2026-02-19T20:41:17.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:17.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:17.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:17.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:17.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:17.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:17.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=30464' 2026-02-19T20:41:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30464 2026-02-19T20:41:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 30464 2026-02-19T20:41:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:17.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:17.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:17.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:17.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:17.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:17.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=30747], [object_number=366, tag_tid=26, entry_tid=30746], [object_number=365, tag_tid=26, entry_tid=30745], [object_number=364, tag_tid=26, entry_tid=30744]]], state=connected] 2026-02-19T20:41:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=30747], [object_number=366, tag_tid=26, entry_tid=30746], [object_number=365, tag_tid=26, entry_tid=30745], [object_number=364, tag_tid=26, entry_tid=30744]]], state=connected] 2026-02-19T20:41:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:17.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=30747' 2026-02-19T20:41:17.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=30747' 2026-02-19T20:41:17.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=30747' '!=' 'object_number=364, tag_tid=26, entry_tid=30464' 2026-02-19T20:41:17.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:17.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=30747' = 'object_number=364, tag_tid=26, entry_tid=30464' 2026-02-19T20:41:17.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=30747' 2026-02-19T20:41:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=30747' 2026-02-19T20:41:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:17.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:17.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=30747' 2026-02-19T20:41:17.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:17.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=30747 2026-02-19T20:41:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 30747 2026-02-19T20:41:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:18.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=31023], [object_number=366, tag_tid=26, entry_tid=31022], [object_number=365, tag_tid=26, entry_tid=31021], [object_number=364, tag_tid=26, entry_tid=31020]]], state=connected] 2026-02-19T20:41:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:18.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:18.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:18.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:18.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=31023], [object_number=366, tag_tid=26, entry_tid=31022], [object_number=365, tag_tid=26, entry_tid=31021], [object_number=364, tag_tid=26, entry_tid=31020]]], state=connected] 2026-02-19T20:41:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=31023' 2026-02-19T20:41:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=31023' 2026-02-19T20:41:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=31023' '!=' 'object_number=367, tag_tid=26, entry_tid=30747' 2026-02-19T20:41:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=31023' = 'object_number=367, tag_tid=26, entry_tid=30747' 2026-02-19T20:41:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=31023' 2026-02-19T20:41:18.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:18.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:18.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:18.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:18.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=31023' 2026-02-19T20:41:18.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:18.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:18.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:18.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=31023' 2026-02-19T20:41:18.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:18.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31023 2026-02-19T20:41:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 31023 2026-02-19T20:41:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:18.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:18.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:18.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:18.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:18.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:18.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:18.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:18.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:18.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:18.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:18.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:18.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:18.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:18.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=31295], [object_number=366, tag_tid=26, entry_tid=31294], [object_number=365, tag_tid=26, entry_tid=31293], [object_number=364, tag_tid=26, entry_tid=31292]]], state=connected] 2026-02-19T20:41:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:18.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=31295], [object_number=366, tag_tid=26, entry_tid=31294], [object_number=365, tag_tid=26, entry_tid=31293], [object_number=364, tag_tid=26, entry_tid=31292]]], state=connected] 2026-02-19T20:41:18.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=31295' 2026-02-19T20:41:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=31295' 2026-02-19T20:41:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=31295' '!=' 'object_number=367, tag_tid=26, entry_tid=31023' 2026-02-19T20:41:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=31295' = 'object_number=367, tag_tid=26, entry_tid=31023' 2026-02-19T20:41:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=31295' 2026-02-19T20:41:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=31295' 2026-02-19T20:41:18.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:18.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:18.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:18.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:18.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:18.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:18.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=31295' 2026-02-19T20:41:18.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:18.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31295 2026-02-19T20:41:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 31295 2026-02-19T20:41:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:18.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:18.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:18.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:18.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:18.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:19.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:19.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:19.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:19.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=31552], [object_number=367, tag_tid=26, entry_tid=31551], [object_number=366, tag_tid=26, entry_tid=31550], [object_number=365, tag_tid=26, entry_tid=31549]]], state=connected] 2026-02-19T20:41:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=31552], [object_number=367, tag_tid=26, entry_tid=31551], [object_number=366, tag_tid=26, entry_tid=31550], [object_number=365, tag_tid=26, entry_tid=31549]]], state=connected] 2026-02-19T20:41:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=364, tag_tid=26, entry_tid=31552' 2026-02-19T20:41:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=364, tag_tid=26, entry_tid=31552' 2026-02-19T20:41:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=31552' '!=' 'object_number=367, tag_tid=26, entry_tid=31295' 2026-02-19T20:41:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=31552' = 'object_number=367, tag_tid=26, entry_tid=31295' 2026-02-19T20:41:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=364, tag_tid=26, entry_tid=31552' 2026-02-19T20:41:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=31552' 2026-02-19T20:41:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=31552' 2026-02-19T20:41:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31552 2026-02-19T20:41:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 31552 2026-02-19T20:41:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:19.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:19.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:19.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630], [object_number=365, tag_tid=26, entry_tid=31629], [object_number=364, tag_tid=26, entry_tid=31628]]], state=connected] 2026-02-19T20:41:19.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:19.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:19.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:19.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:19.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:19.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630], [object_number=365, tag_tid=26, entry_tid=31629], [object_number=364, tag_tid=26, entry_tid=31628]]], state=connected] 2026-02-19T20:41:19.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=31631' 2026-02-19T20:41:19.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=31631' 2026-02-19T20:41:19.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=31631' '!=' 'object_number=364, tag_tid=26, entry_tid=31552' 2026-02-19T20:41:19.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:41:19.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=31631' = 'object_number=364, tag_tid=26, entry_tid=31552' 2026-02-19T20:41:19.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=31631' 2026-02-19T20:41:19.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:19.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:19.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=27 2026-02-19T20:41:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=31631' 2026-02-19T20:41:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:41:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:19.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=26 2026-02-19T20:41:19.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:19.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:19.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:19.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:41:19.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=31631' 2026-02-19T20:41:19.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:41:19.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:41:19.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31631 2026-02-19T20:41:19.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 27 = 26 -a 1 -le 31631 2026-02-19T20:41:19.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:41:19.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:41:19.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:41:19.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:41:19.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:41:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:19.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:41:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:41:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:41:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:41:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:41:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:41:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:41:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:41:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:41:19.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:19.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:19.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:41:19.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:41:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:41:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:41:19.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:41:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:41:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:41:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:41:19.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:19.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:19.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:41:19.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:41:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:41:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:41:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:41:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:41:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:41:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:41:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:41:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:41:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:41:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:41:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:41:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:41:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:41:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 91 2026-02-19T20:41:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 91 2026-02-19T20:41:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:41:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=365, tag_tid=27, entry_tid=1], [object_number=364, tag_tid=27, entry_tid=0], [object_number=367, tag_tid=26, entry_tid=31631], [object_number=366, tag_tid=26, entry_tid=31630]]], state=connected] 2026-02-19T20:41:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:41:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=365, tag_tid=27, entry_tid=1' -a 'object_number=365, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=27, entry_tid=1' 2026-02-19T20:41:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2026-02-19T20:41:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap9 2026-02-19T20:41:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:41:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:41:19.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:41:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2026-02-19T20:41:21.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:41:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:41:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:41:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 17 2026-02-19T20:41:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:41:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:41:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:41:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:41:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:41:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:41:21.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:41:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:40:21 2026 2026-02-19T20:41:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:40:42 2026 2026-02-19T20:41:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:41:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:41:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:41:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:41:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:41:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:41:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n snap8-clean ']' 2026-02-19T20:41:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-19T20:41:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap8-clean 2026-02-19T20:41:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:41:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:41:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:41:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:41:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:41:21.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:41:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:41:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:41:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:41:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:41:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap9 2026-02-19T20:41:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap9 2026-02-19T20:41:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:41:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:41:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:41:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:41:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:41:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:41:22.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:41:23.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:41:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:41:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:23.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name=snap9-clean 2026-02-19T20:41:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap9-clean 2026-02-19T20:41:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:41:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap9-clean 2026-02-19T20:41:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2026-02-19T20:41:24.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror test 2026-02-19T20:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=6 2026-02-19T20:41:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:41:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-19T20:41:25.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 29964288 bytes 2026-02-19T20:41:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 111409152 bytes 2026-02-19T20:41:28.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 190629888 bytes 2026-02-19T20:41:28.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 243476480 bytes 2026-02-19T20:41:29.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 309881856 bytes 2026-02-19T20:41:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 352054272 bytes 2026-02-19T20:41:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 407143424 bytes 2026-02-19T20:41:30.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 407781376 bytes 2026-02-19T20:41:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:41:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:41:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:41:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-19T20:41:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:41:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:41:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:41:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:41:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:41:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-19T20:41:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:41:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local description_pattern= 2026-02-19T20:41:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local service_pattern= 2026-02-19T20:41:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:41:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:41:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-19T20:41:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='test: 2026-02-19T20:41:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: global_id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:41:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+replaying 2026-02-19T20:41:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: replaying, {"bytes_per_second":10198411.47,"entries_behind_primary":885,"entries_per_second":639.5,"non_primary_position":{"entry_tid":30747,"object_number":367,"tag_tid":26},"primary_position":{"entry_tid":1,"object_number":365,"tag_tid":27},"seconds_until_synced":1} 2026-02-19T20:41:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: service: mirror.1 on trial151 2026-02-19T20:41:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:41:17 2026-02-19T20:41:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: peer_sites: 2026-02-19T20:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: name: cluster2 2026-02-19T20:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: state: up+stopped 2026-02-19T20:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: description: local image is primary 2026-02-19T20:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: last_update: 2026-02-19 20:41:18' 2026-02-19T20:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ state: .*up+replaying' 2026-02-19T20:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: state: up+replaying 2026-02-19T20:41:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^ description: .*' 2026-02-19T20:41:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: description: replaying, {"bytes_per_second":10198411.47,"entries_behind_primary":885,"entries_per_second":639.5,"non_primary_position":{"entry_tid":30747,"object_number":367,"tag_tid":26},"primary_position":{"entry_tid":1,"object_number":365,"tag_tid":27},"seconds_until_synced":1} 2026-02-19T20:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep '^up+' 2026-02-19T20:41:31.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:41:31.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep 'service: *mirror..* on ' 2026-02-19T20:41:31.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: service: mirror.1 on trial151 2026-02-19T20:41:31.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update 2026-02-19T20:41:31.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-19T20:41:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:41:17' 2026-02-19T20:41:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-19 20:41:17' 2026-02-19T20:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state_pattern=up+replaying 2026-02-19T20:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'prev_last_update=2026-02-19 20:41:17' 2026-02-19T20:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status 2026-02-19T20:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ CEPH_ARGS= 2026-02-19T20:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-19T20:41:31.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status='OKOKOK142064262mirror.1trial15120.3.0-5336-g802bf111falseOK42104271mirror.2trial15120.3.0-5336-g802bf111trueOK43714374mirror.3trial15120.3.0-5336-g802bf111falseOK44734476mirror.0trial15120.3.0-5336-g802bf111falseOKtest959deb13-de1a-48d9-a341-1f95416f8379up+replayingreplaying, {"bytes_per_second":10198411.47,"entries_behind_primary":885,"entries_per_second":639.5,"non_primary_position":{"entry_tid":30747,"object_number":367,"tag_tid":26},"primary_position":{"entry_tid":1,"object_number":365,"tag_tid":27},"seconds_until_synced":1}42064262mirror.1trial1512026-02-19 20:41:17cluster2a054b997-3cfb-48e6-978a-70d6a85e56deup+stoppedlocal image is primary2026-02-19 20:41:18' 2026-02-19T20:41:31.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_update state 2026-02-19T20:41:31.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-19T20:41:31.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_update='2026-02-19 20:41:17' 2026-02-19T20:41:31.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-19T20:41:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=up+replaying 2026-02-19T20:41:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo up+replaying 2026-02-19T20:41:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep up+replaying 2026-02-19T20:41:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:up+replaying 2026-02-19T20:41:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:41:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap10 2026-02-19T20:41:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap10 2026-02-19T20:41:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:41:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:41:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:41:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap10 2026-02-19T20:41:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:42:00.924 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:42:00.918+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:42:00.924 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:42:00.918+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:42:00.925 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:42:00.918+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00d096c0 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:42:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:42:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:42:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:42:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:42:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:02.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:02.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:02.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:02.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:42:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:42:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:42:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:42:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:42:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:42:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:42:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:42:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:42:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:42:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:42:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:42:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:03.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:03.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:03.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:03.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:03.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:03.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:03.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 96 2026-02-19T20:42:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=387, tag_tid=29, entry_tid=20435], [object_number=386, tag_tid=29, entry_tid=20434], [object_number=385, tag_tid=29, entry_tid=20433], [object_number=384, tag_tid=29, entry_tid=20432]]], state=connected] 2026-02-19T20:42:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 96 2026-02-19T20:42:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=387, tag_tid=29, entry_tid=20435], [object_number=386, tag_tid=29, entry_tid=20434], [object_number=385, tag_tid=29, entry_tid=20433], [object_number=384, tag_tid=29, entry_tid=20432]]], state=connected] 2026-02-19T20:42:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=387, tag_tid=29, entry_tid=20435' 2026-02-19T20:42:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=387, tag_tid=29, entry_tid=20435' 2026-02-19T20:42:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=387, tag_tid=29, entry_tid=20435' '!=' '' 2026-02-19T20:42:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=387, tag_tid=29, entry_tid=20435' = '' 2026-02-19T20:42:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=387, tag_tid=29, entry_tid=20435' 2026-02-19T20:42:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:03.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:03.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=387, tag_tid=29, entry_tid=20435' 2026-02-19T20:42:03.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:03.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:03.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:03.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=387, tag_tid=29, entry_tid=20435' 2026-02-19T20:42:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=20435 2026-02-19T20:42:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 20435 2026-02-19T20:42:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:03.718 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:42:03.710+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:42:03.719 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:42:03.714+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:42:03.719 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:42:03.714+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:42:03.719 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:42:03.714+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00d096c0 2026-02-19T20:42:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:03.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:03.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:03.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:03.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:03.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:03.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:03.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:03.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:03.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 96 2026-02-19T20:42:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=388, tag_tid=29, entry_tid=20764], [object_number=391, tag_tid=29, entry_tid=20763], [object_number=390, tag_tid=29, entry_tid=20762], [object_number=385, tag_tid=29, entry_tid=20761]]], state=connected] 2026-02-19T20:42:04.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 96 2026-02-19T20:42:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=388, tag_tid=29, entry_tid=20764], [object_number=391, tag_tid=29, entry_tid=20763], [object_number=390, tag_tid=29, entry_tid=20762], [object_number=385, tag_tid=29, entry_tid=20761]]], state=connected] 2026-02-19T20:42:04.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=388, tag_tid=29, entry_tid=20764' 2026-02-19T20:42:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=388, tag_tid=29, entry_tid=20764' 2026-02-19T20:42:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=388, tag_tid=29, entry_tid=20764' '!=' 'object_number=387, tag_tid=29, entry_tid=20435' 2026-02-19T20:42:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=388, tag_tid=29, entry_tid=20764' = 'object_number=387, tag_tid=29, entry_tid=20435' 2026-02-19T20:42:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=388, tag_tid=29, entry_tid=20764' 2026-02-19T20:42:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:04.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=388, tag_tid=29, entry_tid=20764' 2026-02-19T20:42:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:04.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=388, tag_tid=29, entry_tid=20764' 2026-02-19T20:42:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=20764 2026-02-19T20:42:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 20764 2026-02-19T20:42:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:04.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:04.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:04.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:04.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:04.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:04.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:04.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:04.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=390, tag_tid=29, entry_tid=21034], [object_number=389, tag_tid=29, entry_tid=21033], [object_number=388, tag_tid=29, entry_tid=21032], [object_number=391, tag_tid=29, entry_tid=21031]]], state=connected] 2026-02-19T20:42:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:04.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:04.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:04.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:04.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:04.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:04.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:04.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=390, tag_tid=29, entry_tid=21034], [object_number=389, tag_tid=29, entry_tid=21033], [object_number=388, tag_tid=29, entry_tid=21032], [object_number=391, tag_tid=29, entry_tid=21031]]], state=connected] 2026-02-19T20:42:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=390, tag_tid=29, entry_tid=21034' 2026-02-19T20:42:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=390, tag_tid=29, entry_tid=21034' 2026-02-19T20:42:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=390, tag_tid=29, entry_tid=21034' '!=' 'object_number=388, tag_tid=29, entry_tid=20764' 2026-02-19T20:42:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=390, tag_tid=29, entry_tid=21034' = 'object_number=388, tag_tid=29, entry_tid=20764' 2026-02-19T20:42:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=390, tag_tid=29, entry_tid=21034' 2026-02-19T20:42:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=390, tag_tid=29, entry_tid=21034' 2026-02-19T20:42:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=390, tag_tid=29, entry_tid=21034' 2026-02-19T20:42:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=21034 2026-02-19T20:42:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 21034 2026-02-19T20:42:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:04.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:04.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:04.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=389, tag_tid=29, entry_tid=21313], [object_number=388, tag_tid=29, entry_tid=21312], [object_number=391, tag_tid=29, entry_tid=21311], [object_number=390, tag_tid=29, entry_tid=21310]]], state=connected] 2026-02-19T20:42:04.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:04.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:04.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:04.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:04.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:04.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:05.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=389, tag_tid=29, entry_tid=21313], [object_number=388, tag_tid=29, entry_tid=21312], [object_number=391, tag_tid=29, entry_tid=21311], [object_number=390, tag_tid=29, entry_tid=21310]]], state=connected] 2026-02-19T20:42:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=389, tag_tid=29, entry_tid=21313' 2026-02-19T20:42:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=389, tag_tid=29, entry_tid=21313' 2026-02-19T20:42:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=389, tag_tid=29, entry_tid=21313' '!=' 'object_number=390, tag_tid=29, entry_tid=21034' 2026-02-19T20:42:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=389, tag_tid=29, entry_tid=21313' = 'object_number=390, tag_tid=29, entry_tid=21034' 2026-02-19T20:42:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=389, tag_tid=29, entry_tid=21313' 2026-02-19T20:42:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:05.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=389, tag_tid=29, entry_tid=21313' 2026-02-19T20:42:05.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:05.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:05.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=389, tag_tid=29, entry_tid=21313' 2026-02-19T20:42:05.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:05.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=21313 2026-02-19T20:42:05.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 21313 2026-02-19T20:42:05.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:05.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:05.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:05.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:05.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:05.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:05.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:05.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:05.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:05.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:05.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:05.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:05.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:05.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:05.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:05.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:05.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:05.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:05.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:05.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:05.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:05.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=389, tag_tid=29, entry_tid=21581], [object_number=388, tag_tid=29, entry_tid=21580], [object_number=391, tag_tid=29, entry_tid=21579], [object_number=390, tag_tid=29, entry_tid=21578]]], state=connected] 2026-02-19T20:42:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:05.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:05.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:05.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:05.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=389, tag_tid=29, entry_tid=21581], [object_number=388, tag_tid=29, entry_tid=21580], [object_number=391, tag_tid=29, entry_tid=21579], [object_number=390, tag_tid=29, entry_tid=21578]]], state=connected] 2026-02-19T20:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=389, tag_tid=29, entry_tid=21581' 2026-02-19T20:42:05.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=389, tag_tid=29, entry_tid=21581' 2026-02-19T20:42:05.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=389, tag_tid=29, entry_tid=21581' '!=' 'object_number=389, tag_tid=29, entry_tid=21313' 2026-02-19T20:42:05.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:05.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=389, tag_tid=29, entry_tid=21581' = 'object_number=389, tag_tid=29, entry_tid=21313' 2026-02-19T20:42:05.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=389, tag_tid=29, entry_tid=21581' 2026-02-19T20:42:05.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:05.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:05.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:05.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=389, tag_tid=29, entry_tid=21581' 2026-02-19T20:42:05.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:05.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=389, tag_tid=29, entry_tid=21581' 2026-02-19T20:42:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=21581 2026-02-19T20:42:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 21581 2026-02-19T20:42:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:05.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:05.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:05.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:05.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:05.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:05.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:05.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:05.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:05.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:05.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:05.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:05.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:05.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:05.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:05.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:05.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:05.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:05.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:05.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=391, tag_tid=29, entry_tid=21847], [object_number=390, tag_tid=29, entry_tid=21846], [object_number=389, tag_tid=29, entry_tid=21845], [object_number=388, tag_tid=29, entry_tid=21844]]], state=connected] 2026-02-19T20:42:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=391, tag_tid=29, entry_tid=21847], [object_number=390, tag_tid=29, entry_tid=21846], [object_number=389, tag_tid=29, entry_tid=21845], [object_number=388, tag_tid=29, entry_tid=21844]]], state=connected] 2026-02-19T20:42:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=391, tag_tid=29, entry_tid=21847' 2026-02-19T20:42:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=391, tag_tid=29, entry_tid=21847' 2026-02-19T20:42:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=391, tag_tid=29, entry_tid=21847' '!=' 'object_number=389, tag_tid=29, entry_tid=21581' 2026-02-19T20:42:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=391, tag_tid=29, entry_tid=21847' = 'object_number=389, tag_tid=29, entry_tid=21581' 2026-02-19T20:42:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=391, tag_tid=29, entry_tid=21847' 2026-02-19T20:42:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=391, tag_tid=29, entry_tid=21847' 2026-02-19T20:42:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:05.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=391, tag_tid=29, entry_tid=21847' 2026-02-19T20:42:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=21847 2026-02-19T20:42:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 21847 2026-02-19T20:42:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:06.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:06.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:06.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:06.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:06.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=390, tag_tid=29, entry_tid=22122], [object_number=389, tag_tid=29, entry_tid=22121], [object_number=388, tag_tid=29, entry_tid=22120], [object_number=391, tag_tid=29, entry_tid=22119]]], state=connected] 2026-02-19T20:42:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:06.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:06.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:06.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=390, tag_tid=29, entry_tid=22122], [object_number=389, tag_tid=29, entry_tid=22121], [object_number=388, tag_tid=29, entry_tid=22120], [object_number=391, tag_tid=29, entry_tid=22119]]], state=connected] 2026-02-19T20:42:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=390, tag_tid=29, entry_tid=22122' 2026-02-19T20:42:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=390, tag_tid=29, entry_tid=22122' 2026-02-19T20:42:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=390, tag_tid=29, entry_tid=22122' '!=' 'object_number=391, tag_tid=29, entry_tid=21847' 2026-02-19T20:42:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=390, tag_tid=29, entry_tid=22122' = 'object_number=391, tag_tid=29, entry_tid=21847' 2026-02-19T20:42:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=390, tag_tid=29, entry_tid=22122' 2026-02-19T20:42:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=390, tag_tid=29, entry_tid=22122' 2026-02-19T20:42:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=390, tag_tid=29, entry_tid=22122' 2026-02-19T20:42:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:06.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=22122 2026-02-19T20:42:06.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 22122 2026-02-19T20:42:06.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:06.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:06.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:06.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:06.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:06.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:06.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:06.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:06.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:06.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:06.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:06.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:06.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:06.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:06.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:06.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:06.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:06.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:06.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:06.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:06.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:06.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:06.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:06.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:06.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:06.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:06.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:06.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:06.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:06.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:06.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:06.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:06.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:06.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:06.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:06.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:06.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:06.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:06.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:06.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=390, tag_tid=29, entry_tid=22434], [object_number=389, tag_tid=29, entry_tid=22433], [object_number=388, tag_tid=29, entry_tid=22432], [object_number=391, tag_tid=29, entry_tid=22431]]], state=connected] 2026-02-19T20:42:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:06.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:06.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=390, tag_tid=29, entry_tid=22434], [object_number=389, tag_tid=29, entry_tid=22433], [object_number=388, tag_tid=29, entry_tid=22432], [object_number=391, tag_tid=29, entry_tid=22431]]], state=connected] 2026-02-19T20:42:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=390, tag_tid=29, entry_tid=22434' 2026-02-19T20:42:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=390, tag_tid=29, entry_tid=22434' 2026-02-19T20:42:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=390, tag_tid=29, entry_tid=22434' '!=' 'object_number=390, tag_tid=29, entry_tid=22122' 2026-02-19T20:42:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=390, tag_tid=29, entry_tid=22434' = 'object_number=390, tag_tid=29, entry_tid=22122' 2026-02-19T20:42:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=390, tag_tid=29, entry_tid=22434' 2026-02-19T20:42:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=390, tag_tid=29, entry_tid=22434' 2026-02-19T20:42:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:06.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:06.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:06.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=390, tag_tid=29, entry_tid=22434' 2026-02-19T20:42:06.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:06.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=22434 2026-02-19T20:42:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 22434 2026-02-19T20:42:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:06.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:06.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:06.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:06.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:06.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:06.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=391, tag_tid=29, entry_tid=22739], [object_number=390, tag_tid=29, entry_tid=22738], [object_number=389, tag_tid=29, entry_tid=22737], [object_number=388, tag_tid=29, entry_tid=22736]]], state=connected] 2026-02-19T20:42:07.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:07.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:07.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:07.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:07.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:07.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=391, tag_tid=29, entry_tid=22739], [object_number=390, tag_tid=29, entry_tid=22738], [object_number=389, tag_tid=29, entry_tid=22737], [object_number=388, tag_tid=29, entry_tid=22736]]], state=connected] 2026-02-19T20:42:07.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=391, tag_tid=29, entry_tid=22739' 2026-02-19T20:42:07.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=391, tag_tid=29, entry_tid=22739' 2026-02-19T20:42:07.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=391, tag_tid=29, entry_tid=22739' '!=' 'object_number=390, tag_tid=29, entry_tid=22434' 2026-02-19T20:42:07.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:07.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=391, tag_tid=29, entry_tid=22739' = 'object_number=390, tag_tid=29, entry_tid=22434' 2026-02-19T20:42:07.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=391, tag_tid=29, entry_tid=22739' 2026-02-19T20:42:07.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:07.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:07.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:07.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:07.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=391, tag_tid=29, entry_tid=22739' 2026-02-19T20:42:07.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=391, tag_tid=29, entry_tid=22739' 2026-02-19T20:42:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:07.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=22739 2026-02-19T20:42:07.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 22739 2026-02-19T20:42:07.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:07.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:07.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:07.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:07.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:07.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:07.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:07.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:07.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:07.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:07.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:07.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:07.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:07.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:07.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:07.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:07.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:07.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:07.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:07.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:07.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:07.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:07.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:07.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:07.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:07.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:07.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:07.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:07.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:07.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:07.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=389, tag_tid=29, entry_tid=23013], [object_number=388, tag_tid=29, entry_tid=23012], [object_number=391, tag_tid=29, entry_tid=23011], [object_number=390, tag_tid=29, entry_tid=23010]]], state=connected] 2026-02-19T20:42:07.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=389, tag_tid=29, entry_tid=23013], [object_number=388, tag_tid=29, entry_tid=23012], [object_number=391, tag_tid=29, entry_tid=23011], [object_number=390, tag_tid=29, entry_tid=23010]]], state=connected] 2026-02-19T20:42:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=389, tag_tid=29, entry_tid=23013' 2026-02-19T20:42:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=389, tag_tid=29, entry_tid=23013' 2026-02-19T20:42:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=389, tag_tid=29, entry_tid=23013' '!=' 'object_number=391, tag_tid=29, entry_tid=22739' 2026-02-19T20:42:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=389, tag_tid=29, entry_tid=23013' = 'object_number=391, tag_tid=29, entry_tid=22739' 2026-02-19T20:42:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=389, tag_tid=29, entry_tid=23013' 2026-02-19T20:42:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:07.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:07.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:07.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=389, tag_tid=29, entry_tid=23013' 2026-02-19T20:42:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:07.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:07.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:07.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:07.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:07.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=389, tag_tid=29, entry_tid=23013' 2026-02-19T20:42:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:07.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23013 2026-02-19T20:42:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 23013 2026-02-19T20:42:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=389, tag_tid=29, entry_tid=23301], [object_number=388, tag_tid=29, entry_tid=23300], [object_number=391, tag_tid=29, entry_tid=23299], [object_number=390, tag_tid=29, entry_tid=23298]]], state=connected] 2026-02-19T20:42:08.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:08.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:08.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:08.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:08.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=389, tag_tid=29, entry_tid=23301], [object_number=388, tag_tid=29, entry_tid=23300], [object_number=391, tag_tid=29, entry_tid=23299], [object_number=390, tag_tid=29, entry_tid=23298]]], state=connected] 2026-02-19T20:42:08.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=389, tag_tid=29, entry_tid=23301' 2026-02-19T20:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=389, tag_tid=29, entry_tid=23301' 2026-02-19T20:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=389, tag_tid=29, entry_tid=23301' '!=' 'object_number=389, tag_tid=29, entry_tid=23013' 2026-02-19T20:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=389, tag_tid=29, entry_tid=23301' = 'object_number=389, tag_tid=29, entry_tid=23013' 2026-02-19T20:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=389, tag_tid=29, entry_tid=23301' 2026-02-19T20:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:08.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:08.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=389, tag_tid=29, entry_tid=23301' 2026-02-19T20:42:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:08.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=389, tag_tid=29, entry_tid=23301' 2026-02-19T20:42:08.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:08.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23301 2026-02-19T20:42:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 23301 2026-02-19T20:42:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:08.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:08.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:08.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:08.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:08.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:08.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:08.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:08.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=388, tag_tid=29, entry_tid=23576], [object_number=391, tag_tid=29, entry_tid=23575], [object_number=390, tag_tid=29, entry_tid=23574], [object_number=389, tag_tid=29, entry_tid=23573]]], state=connected] 2026-02-19T20:42:08.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=388, tag_tid=29, entry_tid=23576], [object_number=391, tag_tid=29, entry_tid=23575], [object_number=390, tag_tid=29, entry_tid=23574], [object_number=389, tag_tid=29, entry_tid=23573]]], state=connected] 2026-02-19T20:42:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=388, tag_tid=29, entry_tid=23576' 2026-02-19T20:42:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=388, tag_tid=29, entry_tid=23576' 2026-02-19T20:42:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=388, tag_tid=29, entry_tid=23576' '!=' 'object_number=389, tag_tid=29, entry_tid=23301' 2026-02-19T20:42:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=388, tag_tid=29, entry_tid=23576' = 'object_number=389, tag_tid=29, entry_tid=23301' 2026-02-19T20:42:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=388, tag_tid=29, entry_tid=23576' 2026-02-19T20:42:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:08.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:08.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=388, tag_tid=29, entry_tid=23576' 2026-02-19T20:42:08.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:08.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:08.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=388, tag_tid=29, entry_tid=23576' 2026-02-19T20:42:08.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:08.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23576 2026-02-19T20:42:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 23576 2026-02-19T20:42:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:08.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:08.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:08.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:08.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:08.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:08.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:08.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:08.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:08.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:08.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:08.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:08.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:08.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:08.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:08.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:08.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:08.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:08.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:08.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:08.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=388, tag_tid=29, entry_tid=23908], [object_number=391, tag_tid=29, entry_tid=23907], [object_number=390, tag_tid=29, entry_tid=23906], [object_number=389, tag_tid=29, entry_tid=23905]]], state=connected] 2026-02-19T20:42:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:08.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=388, tag_tid=29, entry_tid=23908], [object_number=391, tag_tid=29, entry_tid=23907], [object_number=390, tag_tid=29, entry_tid=23906], [object_number=389, tag_tid=29, entry_tid=23905]]], state=connected] 2026-02-19T20:42:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=388, tag_tid=29, entry_tid=23908' 2026-02-19T20:42:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=388, tag_tid=29, entry_tid=23908' 2026-02-19T20:42:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=388, tag_tid=29, entry_tid=23908' '!=' 'object_number=388, tag_tid=29, entry_tid=23576' 2026-02-19T20:42:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=388, tag_tid=29, entry_tid=23908' = 'object_number=388, tag_tid=29, entry_tid=23576' 2026-02-19T20:42:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=388, tag_tid=29, entry_tid=23908' 2026-02-19T20:42:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:08.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=388, tag_tid=29, entry_tid=23908' 2026-02-19T20:42:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:08.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=388, tag_tid=29, entry_tid=23908' 2026-02-19T20:42:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:08.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23908 2026-02-19T20:42:08.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 23908 2026-02-19T20:42:08.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:08.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:08.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:09.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=389, tag_tid=29, entry_tid=24185], [object_number=388, tag_tid=29, entry_tid=24184], [object_number=391, tag_tid=29, entry_tid=24183], [object_number=390, tag_tid=29, entry_tid=24182]]], state=connected] 2026-02-19T20:42:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=389, tag_tid=29, entry_tid=24185], [object_number=388, tag_tid=29, entry_tid=24184], [object_number=391, tag_tid=29, entry_tid=24183], [object_number=390, tag_tid=29, entry_tid=24182]]], state=connected] 2026-02-19T20:42:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=389, tag_tid=29, entry_tid=24185' 2026-02-19T20:42:09.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=389, tag_tid=29, entry_tid=24185' 2026-02-19T20:42:09.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=389, tag_tid=29, entry_tid=24185' '!=' 'object_number=388, tag_tid=29, entry_tid=23908' 2026-02-19T20:42:09.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:09.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=389, tag_tid=29, entry_tid=24185' = 'object_number=388, tag_tid=29, entry_tid=23908' 2026-02-19T20:42:09.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=389, tag_tid=29, entry_tid=24185' 2026-02-19T20:42:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:09.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=389, tag_tid=29, entry_tid=24185' 2026-02-19T20:42:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=389, tag_tid=29, entry_tid=24185' 2026-02-19T20:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:09.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24185 2026-02-19T20:42:09.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 24185 2026-02-19T20:42:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:09.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:09.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:09.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:09.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:09.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:09.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:09.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:09.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:09.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:09.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:09.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:09.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:09.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=390, tag_tid=29, entry_tid=24490], [object_number=389, tag_tid=29, entry_tid=24489], [object_number=388, tag_tid=29, entry_tid=24488], [object_number=391, tag_tid=29, entry_tid=24487]]], state=connected] 2026-02-19T20:42:09.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:09.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=390, tag_tid=29, entry_tid=24490], [object_number=389, tag_tid=29, entry_tid=24489], [object_number=388, tag_tid=29, entry_tid=24488], [object_number=391, tag_tid=29, entry_tid=24487]]], state=connected] 2026-02-19T20:42:09.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=390, tag_tid=29, entry_tid=24490' 2026-02-19T20:42:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=390, tag_tid=29, entry_tid=24490' 2026-02-19T20:42:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=390, tag_tid=29, entry_tid=24490' '!=' 'object_number=389, tag_tid=29, entry_tid=24185' 2026-02-19T20:42:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=390, tag_tid=29, entry_tid=24490' = 'object_number=389, tag_tid=29, entry_tid=24185' 2026-02-19T20:42:09.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=390, tag_tid=29, entry_tid=24490' 2026-02-19T20:42:09.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:09.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:09.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:09.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:09.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=390, tag_tid=29, entry_tid=24490' 2026-02-19T20:42:09.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:09.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:09.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:09.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=390, tag_tid=29, entry_tid=24490' 2026-02-19T20:42:09.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:09.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:09.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24490 2026-02-19T20:42:09.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 24490 2026-02-19T20:42:09.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:09.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:09.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:10.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:10.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:10.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:10.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:10.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:10.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:10.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:10.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=391, tag_tid=29, entry_tid=24763], [object_number=390, tag_tid=29, entry_tid=24762], [object_number=389, tag_tid=29, entry_tid=24761], [object_number=388, tag_tid=29, entry_tid=24760]]], state=connected] 2026-02-19T20:42:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:10.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 97 2026-02-19T20:42:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=391, tag_tid=29, entry_tid=24763], [object_number=390, tag_tid=29, entry_tid=24762], [object_number=389, tag_tid=29, entry_tid=24761], [object_number=388, tag_tid=29, entry_tid=24760]]], state=connected] 2026-02-19T20:42:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=391, tag_tid=29, entry_tid=24763' 2026-02-19T20:42:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=391, tag_tid=29, entry_tid=24763' 2026-02-19T20:42:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=391, tag_tid=29, entry_tid=24763' '!=' 'object_number=390, tag_tid=29, entry_tid=24490' 2026-02-19T20:42:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=391, tag_tid=29, entry_tid=24763' = 'object_number=390, tag_tid=29, entry_tid=24490' 2026-02-19T20:42:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=391, tag_tid=29, entry_tid=24763' 2026-02-19T20:42:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=391, tag_tid=29, entry_tid=24763' 2026-02-19T20:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:10.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=391, tag_tid=29, entry_tid=24763' 2026-02-19T20:42:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24763 2026-02-19T20:42:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 24763 2026-02-19T20:42:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:10.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:10.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:10.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:10.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:10.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:10.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:10.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:10.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:10.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:10.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:10.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:10.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:10.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:10.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:10.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:10.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:10.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:10.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:10.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=25029], [object_number=392, tag_tid=29, entry_tid=25028], [object_number=395, tag_tid=29, entry_tid=25027], [object_number=394, tag_tid=29, entry_tid=25026]]], state=connected] 2026-02-19T20:42:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:10.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:10.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:10.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:10.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:10.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:10.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=25029], [object_number=392, tag_tid=29, entry_tid=25028], [object_number=395, tag_tid=29, entry_tid=25027], [object_number=394, tag_tid=29, entry_tid=25026]]], state=connected] 2026-02-19T20:42:10.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:10.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=29, entry_tid=25029' 2026-02-19T20:42:10.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=393, tag_tid=29, entry_tid=25029' 2026-02-19T20:42:10.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=25029' '!=' 'object_number=391, tag_tid=29, entry_tid=24763' 2026-02-19T20:42:10.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:10.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=25029' = 'object_number=391, tag_tid=29, entry_tid=24763' 2026-02-19T20:42:10.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=393, tag_tid=29, entry_tid=25029' 2026-02-19T20:42:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:10.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=29, entry_tid=25029' 2026-02-19T20:42:10.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:10.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=29, entry_tid=25029' 2026-02-19T20:42:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25029 2026-02-19T20:42:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 25029 2026-02-19T20:42:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:10.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:10.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:10.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:10.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:10.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:10.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:10.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:10.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:11.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:11.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:11.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:11.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:11.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=394, tag_tid=29, entry_tid=25330], [object_number=393, tag_tid=29, entry_tid=25329], [object_number=392, tag_tid=29, entry_tid=25328], [object_number=395, tag_tid=29, entry_tid=25327]]], state=connected] 2026-02-19T20:42:11.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:11.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=394, tag_tid=29, entry_tid=25330], [object_number=393, tag_tid=29, entry_tid=25329], [object_number=392, tag_tid=29, entry_tid=25328], [object_number=395, tag_tid=29, entry_tid=25327]]], state=connected] 2026-02-19T20:42:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=394, tag_tid=29, entry_tid=25330' 2026-02-19T20:42:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=394, tag_tid=29, entry_tid=25330' 2026-02-19T20:42:11.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=394, tag_tid=29, entry_tid=25330' '!=' 'object_number=393, tag_tid=29, entry_tid=25029' 2026-02-19T20:42:11.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:11.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=394, tag_tid=29, entry_tid=25330' = 'object_number=393, tag_tid=29, entry_tid=25029' 2026-02-19T20:42:11.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=394, tag_tid=29, entry_tid=25330' 2026-02-19T20:42:11.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=394, tag_tid=29, entry_tid=25330' 2026-02-19T20:42:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=394, tag_tid=29, entry_tid=25330' 2026-02-19T20:42:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25330 2026-02-19T20:42:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 25330 2026-02-19T20:42:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:11.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:11.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:11.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:11.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:11.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:11.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:11.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:11.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:11.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:11.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:11.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:11.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:11.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:11.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:11.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:11.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:11.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:11.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=25669], [object_number=392, tag_tid=29, entry_tid=25668], [object_number=395, tag_tid=29, entry_tid=25667], [object_number=394, tag_tid=29, entry_tid=25666]]], state=connected] 2026-02-19T20:42:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:11.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=25669], [object_number=392, tag_tid=29, entry_tid=25668], [object_number=395, tag_tid=29, entry_tid=25667], [object_number=394, tag_tid=29, entry_tid=25666]]], state=connected] 2026-02-19T20:42:11.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=29, entry_tid=25669' 2026-02-19T20:42:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=393, tag_tid=29, entry_tid=25669' 2026-02-19T20:42:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=25669' '!=' 'object_number=394, tag_tid=29, entry_tid=25330' 2026-02-19T20:42:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=25669' = 'object_number=394, tag_tid=29, entry_tid=25330' 2026-02-19T20:42:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=393, tag_tid=29, entry_tid=25669' 2026-02-19T20:42:11.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:11.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:11.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:11.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:11.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=29, entry_tid=25669' 2026-02-19T20:42:11.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:11.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:11.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=29, entry_tid=25669' 2026-02-19T20:42:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25669 2026-02-19T20:42:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 25669 2026-02-19T20:42:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:12.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:12.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:12.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:12.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:12.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:12.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:12.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:12.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=395, tag_tid=29, entry_tid=25939], [object_number=394, tag_tid=29, entry_tid=25938], [object_number=393, tag_tid=29, entry_tid=25937], [object_number=392, tag_tid=29, entry_tid=25936]]], state=connected] 2026-02-19T20:42:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:12.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:12.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=395, tag_tid=29, entry_tid=25939], [object_number=394, tag_tid=29, entry_tid=25938], [object_number=393, tag_tid=29, entry_tid=25937], [object_number=392, tag_tid=29, entry_tid=25936]]], state=connected] 2026-02-19T20:42:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=395, tag_tid=29, entry_tid=25939' 2026-02-19T20:42:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=395, tag_tid=29, entry_tid=25939' 2026-02-19T20:42:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=395, tag_tid=29, entry_tid=25939' '!=' 'object_number=393, tag_tid=29, entry_tid=25669' 2026-02-19T20:42:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=395, tag_tid=29, entry_tid=25939' = 'object_number=393, tag_tid=29, entry_tid=25669' 2026-02-19T20:42:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=395, tag_tid=29, entry_tid=25939' 2026-02-19T20:42:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=395, tag_tid=29, entry_tid=25939' 2026-02-19T20:42:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:12.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:12.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=395, tag_tid=29, entry_tid=25939' 2026-02-19T20:42:12.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:12.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:12.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25939 2026-02-19T20:42:12.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 25939 2026-02-19T20:42:12.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:12.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:12.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:12.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:12.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:12.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:12.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:12.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:12.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:12.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:12.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26209], [object_number=392, tag_tid=29, entry_tid=26208], [object_number=395, tag_tid=29, entry_tid=26207], [object_number=394, tag_tid=29, entry_tid=26206]]], state=connected] 2026-02-19T20:42:12.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:12.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26209], [object_number=392, tag_tid=29, entry_tid=26208], [object_number=395, tag_tid=29, entry_tid=26207], [object_number=394, tag_tid=29, entry_tid=26206]]], state=connected] 2026-02-19T20:42:12.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=29, entry_tid=26209' 2026-02-19T20:42:12.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=393, tag_tid=29, entry_tid=26209' 2026-02-19T20:42:12.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=26209' '!=' 'object_number=395, tag_tid=29, entry_tid=25939' 2026-02-19T20:42:12.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:12.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=26209' = 'object_number=395, tag_tid=29, entry_tid=25939' 2026-02-19T20:42:12.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=393, tag_tid=29, entry_tid=26209' 2026-02-19T20:42:12.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:12.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:12.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:12.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:12.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=29, entry_tid=26209' 2026-02-19T20:42:12.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:12.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=29, entry_tid=26209' 2026-02-19T20:42:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26209 2026-02-19T20:42:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 26209 2026-02-19T20:42:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:12.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:12.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:12.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:12.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:12.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:12.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:12.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:12.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:12.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:12.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:12.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:12.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:12.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:12.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:12.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:12.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:12.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:12.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:12.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:12.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:12.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:12.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:12.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:12.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:12.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:12.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:12.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26309], [object_number=392, tag_tid=29, entry_tid=26308], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:12.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:12.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:12.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:12.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:12.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:12.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:12.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:12.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26309], [object_number=392, tag_tid=29, entry_tid=26308], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:13.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=26309' '!=' 'object_number=393, tag_tid=29, entry_tid=26209' 2026-02-19T20:42:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=26309' = 'object_number=393, tag_tid=29, entry_tid=26209' 2026-02-19T20:42:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=30 2026-02-19T20:42:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=29 2026-02-19T20:42:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26309 2026-02-19T20:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 30 = 29 -a 1 -le 26309 2026-02-19T20:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:13.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:13.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:13.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:13.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:13.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:13.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:13.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:13.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:13.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:13.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:13.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:13.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:13.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:13.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:13.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26309], [object_number=392, tag_tid=29, entry_tid=26308], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:13.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:13.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:13.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:13.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26309], [object_number=392, tag_tid=29, entry_tid=26308], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:13.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=26309' '!=' 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:42:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:13.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:13.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:13.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:13.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:13.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:13.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:13.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:13.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:14.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:14.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26309], [object_number=392, tag_tid=29, entry_tid=26308], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:14.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26309], [object_number=392, tag_tid=29, entry_tid=26308], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=26309' '!=' 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:14.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:14.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:14.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26309], [object_number=392, tag_tid=29, entry_tid=26308], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:15.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:15.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:15.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:15.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26309], [object_number=392, tag_tid=29, entry_tid=26308], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:15.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:15.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:15.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=26309' '!=' 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:15.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:15.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:42:16.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:17.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:17.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:17.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:17.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:17.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:17.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26309], [object_number=392, tag_tid=29, entry_tid=26308], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:17.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:17.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:17.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:17.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=29, entry_tid=26309], [object_number=392, tag_tid=29, entry_tid=26308], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:17.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:17.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:17.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=29, entry_tid=26309' '!=' 'object_number=393, tag_tid=29, entry_tid=26309' 2026-02-19T20:42:17.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:17.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:42:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:19.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:19.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:19.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:19.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:19.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:19.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:19.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:19.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:19.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:19.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:19.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=30, entry_tid=1' -a 'object_number=393, tag_tid=30, entry_tid=1' = 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2026-02-19T20:42:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap10 2026-02-19T20:42:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:42:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:42:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:42:20.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2026-02-19T20:42:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:42:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:42:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:42:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 19 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:41:22 2026 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:41:43 2026 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:42:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:42:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:42:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:42:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:42:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n snap9-clean ']' 2026-02-19T20:42:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-19T20:42:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap9-clean 2026-02-19T20:42:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:20.570 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4613395136788079) > weight (0.25) 2026-02-19T20:42:20.570 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7688943383960511) > weight (0.25) 2026-02-19T20:42:20.570 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-19T20:42:20.571 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-19T20:42:20.571 INFO:tasks.rbd_mirror_thrash:waiting for 2.5275882566938526 secs before reviving daemons 2026-02-19T20:42:20.571 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T20:42:20.559+0000 7f0e958ab640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 19444) UID: 1000 2026-02-19T20:42:20.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:20.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:21.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:21.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap10 2026-02-19T20:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap10 2026-02-19T20:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:22.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:23.099 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-19T20:42:23.099 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:42:23.100 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-19T20:42:23.100 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-19T20:42:23.101 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-19T20:42:23.110 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-19T20:42:23.110 INFO:tasks.rbd_mirror_thrash:waiting for 81.7896294125314 secs before thrashing 2026-02-19T20:42:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ clean_snap_name=snap10-clean 2026-02-19T20:42:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror test snap10-clean 2026-02-19T20:42:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:42:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap10-clean 2026-02-19T20:42:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2026-02-19T20:42:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-19T20:42:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-19T20:42:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:42:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:42:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:42:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-19T20:42:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:42:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:42:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:42:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:42:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-19T20:42:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:42:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:42:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:42:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:42:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:42:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:42:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:42:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:42:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:42:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:42:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:42:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:42:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:42:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:24.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:24.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:24.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-19T20:42:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:42:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-19T20:42:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:42:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:42:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:42:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-19T20:42:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:42:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:42:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:42:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:42:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:42:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:42:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:42:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:42:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:42:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:25.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:25.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:25.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:25.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:25.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:25.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:25.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:25.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:25.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:25.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=31, entry_tid=1' -a 'object_number=393, tag_tid=31, entry_tid=1' = 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=30, entry_tid=1' '!=' '' 2026-02-19T20:42:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:42:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=30, entry_tid=1' = '' 2026-02-19T20:42:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:25.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=31 2026-02-19T20:42:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:42:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:25.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=30 2026-02-19T20:42:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:42:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:42:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:42:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=1 2026-02-19T20:42:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 31 = 30 -a 1 -le 1 2026-02-19T20:42:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:42:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:25.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:25.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:25.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:25.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:25.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:25.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:25.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:25.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:25.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:25.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:25.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:25.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:25.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:25.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=31, entry_tid=1' -a 'object_number=393, tag_tid=31, entry_tid=1' = 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=30, entry_tid=1' '!=' 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:42:26.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:26.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:26.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:26.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:26.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:26.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:26.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:26.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:26.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:26.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:26.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:26.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:26.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:26.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:26.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=31, entry_tid=1' -a 'object_number=393, tag_tid=31, entry_tid=1' = 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:26.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=30, entry_tid=1' '!=' 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:26.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:26.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:42:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:27.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:27.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:27.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:27.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:27.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:27.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:27.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:27.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:27.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:27.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:27.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:27.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:27.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:27.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=31, entry_tid=1' -a 'object_number=393, tag_tid=31, entry_tid=1' = 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=30, entry_tid=1' '!=' 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:42:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:29.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:29.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:29.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:29.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:29.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:29.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:29.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:29.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:29.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:29.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:29.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:29.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:29.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:29.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:29.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:29.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:29.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:29.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:29.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:29.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:29.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:29.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:29.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:29.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=31, entry_tid=1' -a 'object_number=393, tag_tid=31, entry_tid=1' = 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:29.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=30, entry_tid=1' '!=' 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:29.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:29.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:42:30.041 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6005312141329633) > weight (0.25) 2026-02-19T20:42:30.041 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3973291610984715) > weight (0.25) 2026-02-19T20:42:30.041 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5505766405501534) > weight (0.25) 2026-02-19T20:42:30.041 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6239816842348535) > weight (0.25) 2026-02-19T20:42:30.041 INFO:tasks.rbd_mirror_thrash:waiting for 76.51858345305708 secs before thrashing 2026-02-19T20:42:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:31.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:31.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:31.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:31.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:31.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:31.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:31.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:31.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:31.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:31.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:31.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:31.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:31.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:31.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:31.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:31.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:31.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:31.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:31.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=30, entry_tid=1], [object_number=392, tag_tid=30, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:31.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:31.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:31.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=31, entry_tid=1' -a 'object_number=393, tag_tid=31, entry_tid=1' = 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:31.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=393, tag_tid=30, entry_tid=1' '!=' 'object_number=393, tag_tid=30, entry_tid=1' 2026-02-19T20:42:31.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:42:31.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:42:33.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror test 2026-02-19T20:42:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:42:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n test ']' 2026-02-19T20:42:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-19T20:42:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-19T20:42:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:42:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:42:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:42:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:42:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:42:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:42:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:42:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-19T20:42:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:42:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:42:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:42:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:42:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:42:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:42:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:42:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:42:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:42:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:42:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-19T20:42:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-19T20:42:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:42:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:42:33.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:42:33.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-19T20:42:33.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:42:33.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:42:33.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:42:33.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:42:33.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:42:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:42:33.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:42:33.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:42:33.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:42:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-19T20:42:33.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-19T20:42:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-19T20:42:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:42:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:33.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:33.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:33.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:33.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:33.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:33.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:33.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-19T20:42:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-19T20:42:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:42:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:42:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:42:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:42:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-19T20:42:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-test.status 2026-02-19T20:42:33.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:42:33.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:33.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-19T20:42:33.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 98 2026-02-19T20:42:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 98 2026-02-19T20:42:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:42:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=393, tag_tid=31, entry_tid=1], [object_number=392, tag_tid=31, entry_tid=0], [object_number=395, tag_tid=29, entry_tid=26307], [object_number=394, tag_tid=29, entry_tid=26306]]], state=connected] 2026-02-19T20:42:33.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.status 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=393, tag_tid=31, entry_tid=1' -a 'object_number=393, tag_tid=31, entry_tid=1' = 'object_number=393, tag_tid=31, entry_tid=1' 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap10-clean 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:42:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:42:34.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'test': 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 20 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a38be6429e 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a38be6429e 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:30:20 2026 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:41:22 2026 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:41:43 2026 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a38be6429e 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 959deb13-de1a-48d9-a341-1f95416f8379 2026-02-19T20:42:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:42:34.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:34.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 10 2026-02-19T20:42:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:42:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap1 2026-02-19T20:42:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap1 2026-02-19T20:42:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap1 2026-02-19T20:42:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:34.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:35.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:35.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap1-clean 2026-02-19T20:42:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-19T20:42:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap1-clean 2026-02-19T20:42:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:36.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:37.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap2 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap2 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap2 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:38.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:38.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap2-clean 2026-02-19T20:42:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-19T20:42:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap2-clean 2026-02-19T20:42:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:39.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:39.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:40.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:42:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap3 2026-02-19T20:42:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap3 2026-02-19T20:42:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap3 2026-02-19T20:42:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:41.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap3-clean 2026-02-19T20:42:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-19T20:42:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap3-clean 2026-02-19T20:42:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:42:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap4 2026-02-19T20:42:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap4 2026-02-19T20:42:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap4 2026-02-19T20:42:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap4-clean 2026-02-19T20:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-19T20:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap4-clean 2026-02-19T20:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:45.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:46.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:46.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap5 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap5 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap5 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:47.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:47.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap5-clean 2026-02-19T20:42:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-19T20:42:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap5-clean 2026-02-19T20:42:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:48.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:48.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:49.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:42:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap6 2026-02-19T20:42:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap6 2026-02-19T20:42:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap6 2026-02-19T20:42:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:50.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap6-clean 2026-02-19T20:42:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-19T20:42:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap6-clean 2026-02-19T20:42:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:51.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:51.579 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:42:51.573+0000 7f9396f28640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-19T20:42:51.579 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:42:51.573+0000 7f9397f2a640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-19T20:42:51.580 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:42:51.573+0000 7f9398f2c640 -1 bdev(0x556e2fad8400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-19T20:42:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:52.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap7 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap7 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap7 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:52.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:52.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:53.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:53.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap7-clean 2026-02-19T20:42:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-19T20:42:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:53.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap7-clean 2026-02-19T20:42:53.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:53.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:53.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:53.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:53.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:54.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:54.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:54.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap8 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap8 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap8 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:55.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:56.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap8-clean 2026-02-19T20:42:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-19T20:42:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap8-clean 2026-02-19T20:42:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:56.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:57.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:58.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:42:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap9 2026-02-19T20:42:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap9 2026-02-19T20:42:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap9 2026-02-19T20:42:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:58.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:42:59.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap9-clean 2026-02-19T20:42:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-19T20:42:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:42:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:42:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap9-clean 2026-02-19T20:42:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:42:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:42:59.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:42:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:42:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:43:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:00.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:43:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap10 2026-02-19T20:43:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap10 2026-02-19T20:43:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap10 2026-02-19T20:43:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:43:00.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:43:00.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:00.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:00.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:43:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:43:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:02.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:43:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:43:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap10-clean 2026-02-19T20:43:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror test snap10-clean 2026-02-19T20:43:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap10-clean 2026-02-19T20:43:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:43:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:43:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:02.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export 2026-02-19T20:43:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:43:02.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-19T20:43:03.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:03.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-test.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-test.export 2026-02-19T20:43:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:43:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 10 2026-02-19T20:43:03.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:03.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap1 2026-02-19T20:43:03.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap1 2026-02-19T20:43:03.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:03.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:03.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:03.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap1 2026-02-19T20:43:03.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2026-02-19T20:43:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:04.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap1-clean 2026-02-19T20:43:04.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2026-02-19T20:43:04.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap1-clean 2026-02-19T20:43:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2026-02-19T20:43:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap2 2026-02-19T20:43:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap2 2026-02-19T20:43:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap2 2026-02-19T20:43:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2026-02-19T20:43:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap2-clean 2026-02-19T20:43:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2026-02-19T20:43:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap2-clean 2026-02-19T20:43:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2026-02-19T20:43:07.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap3 2026-02-19T20:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap3 2026-02-19T20:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap3 2026-02-19T20:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2026-02-19T20:43:08.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap3-clean 2026-02-19T20:43:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2026-02-19T20:43:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:08.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:08.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:08.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap3-clean 2026-02-19T20:43:08.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2026-02-19T20:43:09.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap4 2026-02-19T20:43:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap4 2026-02-19T20:43:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:09.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap4 2026-02-19T20:43:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2026-02-19T20:43:10.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap4-clean 2026-02-19T20:43:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2026-02-19T20:43:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap4-clean 2026-02-19T20:43:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2026-02-19T20:43:11.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:11.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:11.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap5 2026-02-19T20:43:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap5 2026-02-19T20:43:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap5 2026-02-19T20:43:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2026-02-19T20:43:12.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:12.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap5-clean 2026-02-19T20:43:12.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2026-02-19T20:43:12.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:12.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:12.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:12.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap5-clean 2026-02-19T20:43:12.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2026-02-19T20:43:13.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:13.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:13.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap6 2026-02-19T20:43:13.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap6 2026-02-19T20:43:13.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:13.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:13.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:13.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap6 2026-02-19T20:43:13.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2026-02-19T20:43:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:14.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap6-clean 2026-02-19T20:43:14.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2026-02-19T20:43:14.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:14.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:14.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:14.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap6-clean 2026-02-19T20:43:14.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2026-02-19T20:43:15.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap7 2026-02-19T20:43:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap7 2026-02-19T20:43:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap7 2026-02-19T20:43:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2026-02-19T20:43:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:16.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap7-clean 2026-02-19T20:43:16.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2026-02-19T20:43:16.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:16.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:16.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:16.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap7-clean 2026-02-19T20:43:16.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2026-02-19T20:43:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap8 2026-02-19T20:43:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap8 2026-02-19T20:43:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap8 2026-02-19T20:43:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2026-02-19T20:43:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap8-clean 2026-02-19T20:43:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2026-02-19T20:43:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap8-clean 2026-02-19T20:43:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2026-02-19T20:43:19.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap9 2026-02-19T20:43:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap9 2026-02-19T20:43:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap9 2026-02-19T20:43:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2026-02-19T20:43:20.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap9-clean 2026-02-19T20:43:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2026-02-19T20:43:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap9-clean 2026-02-19T20:43:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2026-02-19T20:43:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:21.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T20:43:21.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap10 2026-02-19T20:43:21.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap10 2026-02-19T20:43:21.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:21.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:21.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:21.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap10 2026-02-19T20:43:21.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2026-02-19T20:43:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap10-clean 2026-02-19T20:43:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2026-02-19T20:43:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap10-clean 2026-02-19T20:43:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2026-02-19T20:43:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T20:43:23.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror test 2026-02-19T20:43:23.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:23.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:23.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:23.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T20:43:23.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:43:23.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror test 2026-02-19T20:43:23.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:23.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:23.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:23.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-19T20:43:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-19T20:43:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T20:43:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:43:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-19T20:43:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:43:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T20:43:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T20:43:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:43:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T20:43:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror test 2026-02-19T20:43:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:43:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:43:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:43:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-19T20:43:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T20:43:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=10a38be6429e 2026-02-19T20:43:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T20:43:24.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T20:43:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror test deleted 10a38be6429e 2026-02-19T20:43:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=10a38be6429e 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror test 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:43:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-19T20:43:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T20:43:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=10a38be6429e 2026-02-19T20:43:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 10a38be6429e 2026-02-19T20:43:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 10a38be6429e -o 10a38be6429e = 10a38be6429e 2026-02-19T20:43:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T20:43:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = present 2026-02-19T20:43:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T20:43:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:43:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror test deleted 10a38be6429e 2026-02-19T20:43:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:43:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=test 2026-02-19T20:43:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T20:43:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=10a38be6429e 2026-02-19T20:43:25.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T20:43:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T20:43:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror test 2026-02-19T20:43:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:43:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:43:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=test 2026-02-19T20:43:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-19T20:43:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T20:43:25.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-19T20:43:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T20:43:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T20:43:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T20:43:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:43:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ testlog 'TEST: create many images' 2026-02-19T20:43:25.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ tee -a /tmp/tmp.xpxdoJdHT7/rbd-mirror.test.log 2026-02-19T20:43:25.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ date '+%F %T' 2026-02-19T20:43:25.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo 2026-02-19 20:43:25 TEST: create many images 2026-02-19T20:43:25.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:2026-02-19 20:43:25 TEST: create many images 2026-02-19T20:43:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ snap_name=snap 2026-02-19T20:43:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 50 2026-02-19T20:43:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:43:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_1 2026-02-19T20:43:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 journal 128M 2026-02-19T20:43:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:25.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_1 128M 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:43:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:43:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:43:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:43:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_1 2026-02-19T20:43:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:43:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:43:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:43:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:43:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:43:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-19T20:43:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:43:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:43:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=12 2026-02-19T20:43:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:43:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:43:26.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 82511872 bytes 2026-02-19T20:43:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 154518528 bytes 2026-02-19T20:43:28.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 178960384 bytes 2026-02-19T20:43:29.153 INFO:tasks.ceph.cluster2.osd.1.trial151.stderr:2026-02-19T20:43:29.139+0000 7f58ec161640 -1 reset not still connected to 0x55616b0b55f0 2026-02-19T20:43:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 204306432 bytes 2026-02-19T20:43:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 231985152 bytes 2026-02-19T20:43:29.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 256770048 bytes 2026-02-19T20:43:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 372783104 bytes 2026-02-19T20:43:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 389482496 bytes 2026-02-19T20:43:31.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 417869824 bytes 2026-02-19T20:43:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 472859648 bytes 2026-02-19T20:43:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 504909824 bytes 2026-02-19T20:43:33.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 581313536 bytes 2026-02-19T20:43:33.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 622942208 bytes 2026-02-19T20:43:34.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 629901312 bytes 2026-02-19T20:43:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 653872128 bytes 2026-02-19T20:43:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 686652416 bytes 2026-02-19T20:43:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 775997440 bytes 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_2 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 journal 128M 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_2 128M 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_2 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:43:37.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:43:37.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=11 2026-02-19T20:43:37.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:43:37.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:43:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 82706432 bytes 2026-02-19T20:43:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 137550848 bytes 2026-02-19T20:43:42.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 211606528 bytes 2026-02-19T20:43:42.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 223667200 bytes 2026-02-19T20:43:43.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 299688960 bytes 2026-02-19T20:43:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 402939904 bytes 2026-02-19T20:43:44.901 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.36158857664332544) > weight (0.25) 2026-02-19T20:43:44.901 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6413723166608037) > weight (0.25) 2026-02-19T20:43:44.901 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5570754186393502) > weight (0.25) 2026-02-19T20:43:44.901 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6734621642896831) > weight (0.25) 2026-02-19T20:43:44.901 INFO:tasks.rbd_mirror_thrash:waiting for 92.93340689748902 secs before thrashing 2026-02-19T20:43:45.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 464098304 bytes 2026-02-19T20:43:46.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 488254464 bytes 2026-02-19T20:43:46.561 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9656669957172969) > weight (0.25) 2026-02-19T20:43:46.561 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8846349375240503) > weight (0.25) 2026-02-19T20:43:46.561 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.60374126232977) > weight (0.25) 2026-02-19T20:43:46.561 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-19T20:43:46.562 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-19T20:43:46.562 INFO:tasks.rbd_mirror_thrash:waiting for 5.639220433644919 secs before reviving daemons 2026-02-19T20:43:46.562 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T20:43:46.551+0000 7ff8812f3640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 19554) UID: 1000 2026-02-19T20:43:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 586438656 bytes 2026-02-19T20:43:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:43:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:43:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:43:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:43:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_3 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 journal 128M 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_3 128M 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_3 2026-02-19T20:43:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:43:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:43:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:43:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:43:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:43:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-19T20:43:48.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:43:48.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:43:48.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=7 2026-02-19T20:43:48.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:43:48.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:43:49.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 25283584 bytes 2026-02-19T20:43:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 74656768 bytes 2026-02-19T20:43:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 122602496 bytes 2026-02-19T20:43:51.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 164190208 bytes 2026-02-19T20:43:52.202 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-19T20:43:52.202 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:43:52.203 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-19T20:43:52.203 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-19T20:43:52.203 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-19T20:43:52.205 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-19T20:43:52.205 INFO:tasks.rbd_mirror_thrash:waiting for 116.09294273184634 secs before thrashing 2026-02-19T20:43:52.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 240814080 bytes 2026-02-19T20:43:53.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 260373504 bytes 2026-02-19T20:43:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 290506752 bytes 2026-02-19T20:43:54.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 401736704 bytes 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_4 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 journal 128M 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_4 128M 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:43:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_4 2026-02-19T20:43:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:43:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:43:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:43:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:43:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:43:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-19T20:43:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:43:55.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:43:55.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:43:55.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:43:55.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=9 2026-02-19T20:43:55.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:43:55.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:43:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 103654400 bytes 2026-02-19T20:43:57.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 117481472 bytes 2026-02-19T20:43:58.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 151982080 bytes 2026-02-19T20:43:59.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 171663360 bytes 2026-02-19T20:43:59.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 197016576 bytes 2026-02-19T20:44:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 224442368 bytes 2026-02-19T20:44:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 271915008 bytes 2026-02-19T20:44:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 314118144 bytes 2026-02-19T20:44:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 331675648 bytes 2026-02-19T20:44:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 450989056 bytes 2026-02-19T20:44:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 500281344 bytes 2026-02-19T20:44:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:44:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_5 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 journal 128M 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_5 128M 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_5 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:44:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:44:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=5 2026-02-19T20:44:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:44:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:44:05.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 42675200 bytes 2026-02-19T20:44:06.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 91426816 bytes 2026-02-19T20:44:06.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 131590144 bytes 2026-02-19T20:44:07.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 166287360 bytes 2026-02-19T20:44:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 201500672 bytes 2026-02-19T20:44:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 277186560 bytes 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_6 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 journal 128M 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_6 128M 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_6 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:44:09.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:44:09.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=7 2026-02-19T20:44:09.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:44:09.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:44:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 6234112 bytes 2026-02-19T20:44:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 79263744 bytes 2026-02-19T20:44:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 137202688 bytes 2026-02-19T20:44:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 191002624 bytes 2026-02-19T20:44:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 214446080 bytes 2026-02-19T20:44:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 251986944 bytes 2026-02-19T20:44:16.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 323118080 bytes 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_7 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 journal 128M 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_7 128M 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:44:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:44:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:44:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:44:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_7 2026-02-19T20:44:16.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:44:16.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:16.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:44:16.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:44:16.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:44:16.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:44:16.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-19T20:44:16.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:16.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:16.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:44:16.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:44:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=10 2026-02-19T20:44:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:44:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:44:17.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 22246400 bytes 2026-02-19T20:44:18.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 89293824 bytes 2026-02-19T20:44:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 110007296 bytes 2026-02-19T20:44:20.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 152157184 bytes 2026-02-19T20:44:20.955 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:20.948+0000 7f93a8cad640 -1 reset not still connected to 0x556e336b9c70 2026-02-19T20:44:20.955 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:20.948+0000 7f93a8cad640 -1 reset not still connected to 0x556e33e3f450 2026-02-19T20:44:21.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 204430336 bytes 2026-02-19T20:44:22.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 291006464 bytes 2026-02-19T20:44:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 304733184 bytes 2026-02-19T20:44:24.456 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:24.448+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:44:24.456 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:24.448+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:44:24.456 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:24.448+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00951ba0 2026-02-19T20:44:24.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 418103296 bytes 2026-02-19T20:44:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 468742144 bytes 2026-02-19T20:44:26.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 503744512 bytes 2026-02-19T20:44:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:44:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_8 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 journal 128M 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:44:26.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_8 128M 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:44:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:44:26.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:26.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_8 2026-02-19T20:44:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:44:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:44:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:44:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:44:26.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:44:26.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-19T20:44:26.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:26.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:26.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:44:26.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:44:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=9 2026-02-19T20:44:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:44:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:44:27.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 15702016 bytes 2026-02-19T20:44:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 58017792 bytes 2026-02-19T20:44:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 74747904 bytes 2026-02-19T20:44:30.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 169747456 bytes 2026-02-19T20:44:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 235761664 bytes 2026-02-19T20:44:32.188 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:32.180+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f341a0 2026-02-19T20:44:32.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 337290240 bytes 2026-02-19T20:44:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 392871936 bytes 2026-02-19T20:44:34.241 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:44:34.232+0000 7fd63ae12640 -1 reset not still connected to 0x55f11003bc70 2026-02-19T20:44:34.241 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:44:34.232+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74a90 2026-02-19T20:44:34.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 455248896 bytes 2026-02-19T20:44:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 491577344 bytes 2026-02-19T20:44:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 531795968 bytes 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_9 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 journal 128M 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:44:35.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_9 128M 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:35.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_9 2026-02-19T20:44:35.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:44:35.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:35.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:44:35.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:44:35.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:44:35.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:44:35.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-19T20:44:35.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:35.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:35.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:44:35.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:44:35.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=5 2026-02-19T20:44:35.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:44:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:44:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 95433728 bytes 2026-02-19T20:44:37.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 107062272 bytes 2026-02-19T20:44:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 107656192 bytes 2026-02-19T20:44:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 157476864 bytes 2026-02-19T20:44:39.103 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:39.096+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:44:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 183734272 bytes 2026-02-19T20:44:40.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 254411776 bytes 2026-02-19T20:44:40.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_10 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 journal 128M 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_10 128M 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:44:40.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:44:40.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:44:40.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:40.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_10 2026-02-19T20:44:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:44:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:44:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:44:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:44:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:44:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-19T20:44:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:44:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:44:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=13 2026-02-19T20:44:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:44:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:44:41.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 34743296 bytes 2026-02-19T20:44:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 60895232 bytes 2026-02-19T20:44:42.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 76531712 bytes 2026-02-19T20:44:42.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 95638528 bytes 2026-02-19T20:44:44.250 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:44.244+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2ca90 2026-02-19T20:44:44.251 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:44.244+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f341a0 2026-02-19T20:44:44.672 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:44:44.659+0000 7f5ce3652640 -1 reset not still connected to 0x56104f0db2b0 2026-02-19T20:44:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 200441856 bytes 2026-02-19T20:44:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 202591232 bytes 2026-02-19T20:44:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 203220992 bytes 2026-02-19T20:44:45.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 236537856 bytes 2026-02-19T20:44:45.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 242023424 bytes 2026-02-19T20:44:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 304232448 bytes 2026-02-19T20:44:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 361731072 bytes 2026-02-19T20:44:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 473131008 bytes 2026-02-19T20:44:49.290 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:49.284+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:44:49.290 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:49.284+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fc8f0 2026-02-19T20:44:49.290 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:49.284+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa1a0 2026-02-19T20:44:49.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 473730048 bytes 2026-02-19T20:44:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 477923328 bytes 2026-02-19T20:44:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 519609344 bytes 2026-02-19T20:44:51.229 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:51.224+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:44:51.230 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:51.224+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:44:51.230 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:51.224+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00951ba0 2026-02-19T20:44:51.387 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:44:51.384+0000 7fd63ae12640 -1 reset not still connected to 0x55f10fae8410 2026-02-19T20:44:51.388 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:44:51.384+0000 7fd63ae12640 -1 reset not still connected to 0x55f116e4a270 2026-02-19T20:44:51.388 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:44:51.384+0000 7fd63ae12640 -1 reset not still connected to 0x55f116e4bba0 2026-02-19T20:44:51.388 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:44:51.384+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a3a00 2026-02-19T20:44:51.806 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:44:51.800+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74340 2026-02-19T20:44:51.806 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:44:51.800+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a3ba0 2026-02-19T20:44:51.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 600247296 bytes 2026-02-19T20:44:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 610826240 bytes 2026-02-19T20:44:52.270 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:52.268+0000 7f93a8cad640 -1 reset not still connected to 0x556e336b9c70 2026-02-19T20:44:52.271 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:52.268+0000 7f93a8cad640 -1 reset not still connected to 0x556e33e3f450 2026-02-19T20:44:52.271 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:52.268+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2680 2026-02-19T20:44:52.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 628031488 bytes 2026-02-19T20:44:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 661578752 bytes 2026-02-19T20:44:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:44:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:44:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:44:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:44:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:44:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_11 2026-02-19T20:44:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 journal 128M 2026-02-19T20:44:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_11 128M 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:53.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:44:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:44:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:44:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:44:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_11 2026-02-19T20:44:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:44:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:44:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:44:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:44:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:44:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_11 2026-02-19T20:44:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:44:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:44:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=6 2026-02-19T20:44:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:44:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:44:53.977 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:53.972+0000 7f93a8cad640 -1 reset not still connected to 0x556e324ee9c0 2026-02-19T20:44:53.977 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:53.972+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:44:53.977 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:53.972+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fc8f0 2026-02-19T20:44:53.977 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:44:53.972+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa1a0 2026-02-19T20:44:54.592 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:54.587+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00951ba0 2026-02-19T20:44:54.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 44389376 bytes 2026-02-19T20:44:54.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 82557952 bytes 2026-02-19T20:44:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 96942080 bytes 2026-02-19T20:44:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 111796224 bytes 2026-02-19T20:44:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 150758400 bytes 2026-02-19T20:44:56.309 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:56.303+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c8f0 2026-02-19T20:44:56.310 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:56.303+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2ca90 2026-02-19T20:44:56.311 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:56.307+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f341a0 2026-02-19T20:44:56.467 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:56.463+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:44:56.467 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:56.463+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:44:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 197056512 bytes 2026-02-19T20:44:57.910 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:57.907+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2ca90 2026-02-19T20:44:57.911 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:44:57.907+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f341a0 2026-02-19T20:44:58.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 248521728 bytes 2026-02-19T20:44:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 298457088 bytes 2026-02-19T20:44:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 330001408 bytes 2026-02-19T20:44:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:44:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:44:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:44:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:44:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:44:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_12 2026-02-19T20:44:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 journal 128M 2026-02-19T20:44:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_12 128M 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:44:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:44:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:44:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:44:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:44:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_12 2026-02-19T20:45:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:45:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:45:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:45:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:45:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:45:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_12 2026-02-19T20:45:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:45:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:45:00.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=9 2026-02-19T20:45:00.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:45:00.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:45:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 12264448 bytes 2026-02-19T20:45:00.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 55174144 bytes 2026-02-19T20:45:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 74712064 bytes 2026-02-19T20:45:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 91966464 bytes 2026-02-19T20:45:02.219 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:45:02.215+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a3a00 2026-02-19T20:45:02.325 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:45:02.319+0000 7fd63ae12640 -1 reset not still connected to 0x55f11003bc70 2026-02-19T20:45:02.325 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:45:02.319+0000 7fd63ae12640 -1 reset not still connected to 0x55f11094b110 2026-02-19T20:45:02.325 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:45:02.319+0000 7fd63ae12640 -1 reset not still connected to 0x55f112563c70 2026-02-19T20:45:02.326 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:45:02.319+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74a90 2026-02-19T20:45:03.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 163054592 bytes 2026-02-19T20:45:04.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 234865664 bytes 2026-02-19T20:45:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 253702144 bytes 2026-02-19T20:45:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 283689984 bytes 2026-02-19T20:45:05.412 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:05.407+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa1a0 2026-02-19T20:45:05.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 351826944 bytes 2026-02-19T20:45:06.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 374626304 bytes 2026-02-19T20:45:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 484994048 bytes 2026-02-19T20:45:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 500788224 bytes 2026-02-19T20:45:08.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 539888640 bytes 2026-02-19T20:45:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 549336064 bytes 2026-02-19T20:45:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:45:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:45:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:45:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:45:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:45:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_13 2026-02-19T20:45:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 journal 128M 2026-02-19T20:45:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:45:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:45:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:45:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_13 128M 2026-02-19T20:45:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:45:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:45:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:45:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:45:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_13 2026-02-19T20:45:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:45:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:45:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:45:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:45:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:45:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_13 2026-02-19T20:45:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:45:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:45:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=10 2026-02-19T20:45:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:45:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:45:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 2878464 bytes 2026-02-19T20:45:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 42679296 bytes 2026-02-19T20:45:10.513 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:10.507+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aad00 2026-02-19T20:45:10.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 78518272 bytes 2026-02-19T20:45:10.695 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:10.691+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00950d00 2026-02-19T20:45:10.695 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:10.691+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0340 2026-02-19T20:45:10.696 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:10.691+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043380 2026-02-19T20:45:10.696 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:10.691+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c8f0 2026-02-19T20:45:10.696 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:10.691+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2ca90 2026-02-19T20:45:10.696 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:10.691+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f341a0 2026-02-19T20:45:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 141800448 bytes 2026-02-19T20:45:12.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 156203008 bytes 2026-02-19T20:45:12.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 190871552 bytes 2026-02-19T20:45:13.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 210002944 bytes 2026-02-19T20:45:14.041 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:14.035+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f341a0 2026-02-19T20:45:14.211 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:14.207+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2ca90 2026-02-19T20:45:14.211 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:14.207+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f341a0 2026-02-19T20:45:14.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 259938304 bytes 2026-02-19T20:45:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 354216960 bytes 2026-02-19T20:45:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 398926848 bytes 2026-02-19T20:45:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 445027328 bytes 2026-02-19T20:45:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 489234432 bytes 2026-02-19T20:45:17.836 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5038750367176021) > weight (0.25) 2026-02-19T20:45:17.836 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5213957475729429) > weight (0.25) 2026-02-19T20:45:17.836 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8128593191458336) > weight (0.25) 2026-02-19T20:45:17.836 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-19T20:45:17.837 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-19T20:45:17.837 INFO:tasks.rbd_mirror_thrash:waiting for 5.009359749518243 secs before reviving daemons 2026-02-19T20:45:17.838 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T20:45:17.827+0000 7fdcee5ef640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 38092) UID: 1000 2026-02-19T20:45:17.846 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T20:45:17.835+0000 7fdce75e1640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55780b809c00 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-19T20:45:17.846 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T20:45:17.835+0000 7fdce75e1640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55780b809c00 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-19T20:45:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 509207552 bytes 2026-02-19T20:45:18.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 539275264 bytes 2026-02-19T20:45:18.789 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:45:18.783+0000 7fd63ae12640 -1 reset not still connected to 0x55f11024d930 2026-02-19T20:45:18.790 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:45:18.783+0000 7fd63ae12640 -1 reset not still connected to 0x55f110e16270 2026-02-19T20:45:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:45:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:45:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:45:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_14 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 journal 128M 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_14 128M 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_14 2026-02-19T20:45:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:45:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:45:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:45:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:45:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:45:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_14 2026-02-19T20:45:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:45:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:45:19.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=14 2026-02-19T20:45:19.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:45:19.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:45:19.724 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:19.719+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:45:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 68387840 bytes 2026-02-19T20:45:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 85344256 bytes 2026-02-19T20:45:21.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 155994112 bytes 2026-02-19T20:45:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 168903680 bytes 2026-02-19T20:45:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 234670080 bytes 2026-02-19T20:45:22.847 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-19T20:45:22.847 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:45:22.847 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-19T20:45:22.848 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-19T20:45:22.848 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-19T20:45:22.849 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-19T20:45:22.849 INFO:tasks.rbd_mirror_thrash:waiting for 63.66596114659068 secs before thrashing 2026-02-19T20:45:23.251 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:45:23.239+0000 7efcbc9bc640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x560a7ff86700 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-19T20:45:23.251 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:45:23.239+0000 7efcbc9bc640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x560a7ff86700 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-19T20:45:23.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 270601216 bytes 2026-02-19T20:45:23.702 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:45:23.691+0000 7f7849c13640 -1 bdev(0x557edd010400 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2026-02-19T20:45:23.703 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:45:23.691+0000 7f784dc1b640 -1 bdev(0x557edd010400 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2026-02-19T20:45:23.709 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:45:23.699+0000 7f784cc19640 -1 bdev(0x557edd010400 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2026-02-19T20:45:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 348414976 bytes 2026-02-19T20:45:24.605 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:24.599+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:45:24.605 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:24.599+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:45:24.605 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:24.599+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:45:24.605 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:24.599+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:45:24.605 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:24.599+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:45:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 401755136 bytes 2026-02-19T20:45:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 404897792 bytes 2026-02-19T20:45:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 423268352 bytes 2026-02-19T20:45:26.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 445774848 bytes 2026-02-19T20:45:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 486677504 bytes 2026-02-19T20:45:28.427 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:45:28.415+0000 7f785c199640 -1 reset not still connected to 0x557f199e0dd0 2026-02-19T20:45:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 547628032 bytes 2026-02-19T20:45:28.983 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:28.979+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:45:29.067 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.063+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:45:29.068 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.063+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:45:29.068 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.063+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:45:29.068 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.063+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:45:29.068 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.063+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:45:29.114 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.111+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec3380 2026-02-19T20:45:29.114 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.111+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:45:29.114 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.111+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:45:29.115 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.111+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:45:29.115 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.111+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:45:29.115 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.111+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:45:29.115 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.111+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:45:29.323 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.319+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97450 2026-02-19T20:45:29.324 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.319+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec3380 2026-02-19T20:45:29.324 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.319+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:45:29.324 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.319+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:45:29.324 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.319+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:45:29.324 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.319+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:45:29.324 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.319+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:45:29.325 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:29.319+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:45:29.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 585183232 bytes 2026-02-19T20:45:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 615025664 bytes 2026-02-19T20:45:30.381 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:30.375+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec3380 2026-02-19T20:45:30.381 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:30.375+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:45:30.381 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:30.375+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:45:30.381 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:30.375+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:45:30.381 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:30.375+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:45:30.381 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:30.375+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:45:30.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 666907648 bytes 2026-02-19T20:45:31.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 706343936 bytes 2026-02-19T20:45:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 754970624 bytes 2026-02-19T20:45:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 787675136 bytes 2026-02-19T20:45:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_15 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 journal 128M 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_15 128M 2026-02-19T20:45:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:45:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:45:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:45:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:45:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_15 2026-02-19T20:45:33.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:45:33.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:33.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:45:33.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:45:33.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:45:33.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:45:33.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_15 2026-02-19T20:45:33.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:33.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:33.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:45:33.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:45:33.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=13 2026-02-19T20:45:33.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:45:33.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:45:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 36521984 bytes 2026-02-19T20:45:34.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 56953856 bytes 2026-02-19T20:45:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 78018560 bytes 2026-02-19T20:45:34.913 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:34.907+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:45:34.913 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:34.907+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:45:34.913 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:34.907+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:45:34.914 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:34.907+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:45:34.914 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:34.911+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29a00 2026-02-19T20:45:34.915 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:34.911+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:45:34.915 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:34.911+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:45:35.312 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:35.307+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:45:35.312 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:35.307+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:45:35.312 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:35.307+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:45:35.312 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:35.307+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:45:35.313 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:35.307+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29a00 2026-02-19T20:45:35.314 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:35.307+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:45:35.314 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:35.307+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:45:35.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 112048128 bytes 2026-02-19T20:45:35.989 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:35.983+0000 7f93a8cad640 -1 reset not still connected to 0x556e4483b860 2026-02-19T20:45:35.989 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:35.983+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:45:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 182057984 bytes 2026-02-19T20:45:37.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 262654976 bytes 2026-02-19T20:45:38.041 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:38.035+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:45:38.994 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:38.987+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:45:38.995 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:38.991+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29a00 2026-02-19T20:45:38.995 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:38.991+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:45:38.995 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:38.991+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:45:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 309506048 bytes 2026-02-19T20:45:39.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 339576832 bytes 2026-02-19T20:45:39.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 344852480 bytes 2026-02-19T20:45:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 359697408 bytes 2026-02-19T20:45:40.120 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:40.115+0000 7f93a8cad640 -1 reset not still connected to 0x556e336b8750 2026-02-19T20:45:40.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:40.115+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb2a90 2026-02-19T20:45:40.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:40.115+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb3790 2026-02-19T20:45:40.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:40.115+0000 7f93a8cad640 -1 reset not still connected to 0x556e4483b860 2026-02-19T20:45:40.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:40.115+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ed00 2026-02-19T20:45:40.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:40.115+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:45:40.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:40.115+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:45:40.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:40.115+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:45:40.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:40.115+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:45:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 389279744 bytes 2026-02-19T20:45:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 429742080 bytes 2026-02-19T20:45:41.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 482271232 bytes 2026-02-19T20:45:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 497543168 bytes 2026-02-19T20:45:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 522939392 bytes 2026-02-19T20:45:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 526896128 bytes 2026-02-19T20:45:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 557050880 bytes 2026-02-19T20:45:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 557662208 bytes 2026-02-19T20:45:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 610459648 bytes 2026-02-19T20:45:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2000 IOs, 620815360 bytes 2026-02-19T20:45:44.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2100 IOs, 621421568 bytes 2026-02-19T20:45:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2200 IOs, 684419072 bytes 2026-02-19T20:45:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2300 IOs, 743911424 bytes 2026-02-19T20:45:46.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:45:46.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:45:46.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_16 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 journal 128M 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_16 128M 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:45:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_16 2026-02-19T20:45:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:45:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:45:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:45:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:45:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:45:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_16 2026-02-19T20:45:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:45:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:45:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=13 2026-02-19T20:45:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:45:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:45:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 941056 bytes 2026-02-19T20:45:46.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 36303872 bytes 2026-02-19T20:45:48.212 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:48.207+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:45:48.214 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:48.211+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:45:48.214 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:48.211+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:45:48.215 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:48.211+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:45:48.299 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9942263905296491) > weight (0.25) 2026-02-19T20:45:48.300 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.688057758087199) > weight (0.25) 2026-02-19T20:45:48.300 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8669488330643607) > weight (0.25) 2026-02-19T20:45:48.300 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8236325070971734) > weight (0.25) 2026-02-19T20:45:48.300 INFO:tasks.rbd_mirror_thrash:waiting for 73.59994678146764 secs before thrashing 2026-02-19T20:45:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 141438976 bytes 2026-02-19T20:45:48.375 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:48.371+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:45:49.732 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:45:49.727+0000 7fd63ae12640 -1 reset not still connected to 0x55f110e16270 2026-02-19T20:45:49.732 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:45:49.727+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:45:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 236816384 bytes 2026-02-19T20:45:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 323911680 bytes 2026-02-19T20:45:52.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 411577344 bytes 2026-02-19T20:45:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 442613760 bytes 2026-02-19T20:45:54.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 472728576 bytes 2026-02-19T20:45:54.544 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:54.539+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:45:54.546 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:54.543+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:45:54.546 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:54.543+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:45:54.546 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:54.543+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:45:54.546 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:54.543+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:45:54.546 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:45:54.543+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:45:54.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 511719424 bytes 2026-02-19T20:45:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 519576576 bytes 2026-02-19T20:45:55.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 572667904 bytes 2026-02-19T20:45:55.953 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:45:55.947+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aab60 2026-02-19T20:45:57.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 637769728 bytes 2026-02-19T20:45:58.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 666545152 bytes 2026-02-19T20:45:58.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 725138432 bytes 2026-02-19T20:45:59.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:45:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:45:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:45:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:45:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:45:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_17 2026-02-19T20:45:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 journal 128M 2026-02-19T20:45:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_17 128M 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:45:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:45:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:45:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:45:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_17 2026-02-19T20:45:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:45:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:45:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:45:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:45:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:45:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_17 2026-02-19T20:45:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:45:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:45:59.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:45:59.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:45:59.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=11 2026-02-19T20:45:59.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:45:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:46:00.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 37663744 bytes 2026-02-19T20:46:00.409 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:00.403+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec3380 2026-02-19T20:46:00.409 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:00.403+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:46:00.409 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:00.403+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:46:00.409 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:00.403+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:46:00.409 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:00.403+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:46:00.409 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:00.403+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:46:00.409 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:00.403+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:46:00.749 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:00.743+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:46:00.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 103712768 bytes 2026-02-19T20:46:02.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 160269312 bytes 2026-02-19T20:46:02.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 188872704 bytes 2026-02-19T20:46:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 202941440 bytes 2026-02-19T20:46:03.638 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:46:03.631+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74340 2026-02-19T20:46:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 257313792 bytes 2026-02-19T20:46:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 257937408 bytes 2026-02-19T20:46:04.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 316979200 bytes 2026-02-19T20:46:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 338460672 bytes 2026-02-19T20:46:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 401284096 bytes 2026-02-19T20:46:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 448319488 bytes 2026-02-19T20:46:07.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 466841600 bytes 2026-02-19T20:46:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 520964096 bytes 2026-02-19T20:46:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 532463616 bytes 2026-02-19T20:46:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 574097408 bytes 2026-02-19T20:46:09.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 595318784 bytes 2026-02-19T20:46:10.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_18 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 journal 128M 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_18 128M 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:46:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:46:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:46:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_18 2026-02-19T20:46:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:46:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:46:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:46:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:46:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:46:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_18 2026-02-19T20:46:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:46:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:46:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=10 2026-02-19T20:46:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:46:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:46:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 628736 bytes 2026-02-19T20:46:11.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 37537792 bytes 2026-02-19T20:46:11.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 59194368 bytes 2026-02-19T20:46:12.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 85819392 bytes 2026-02-19T20:46:12.798 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:12.795+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:46:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 132581376 bytes 2026-02-19T20:46:14.154 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:14.151+0000 7fe4dd68c640 -1 reset not still connected to 0x564a08249a00 2026-02-19T20:46:14.155 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:14.151+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2cc30 2026-02-19T20:46:14.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 170729472 bytes 2026-02-19T20:46:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 195185664 bytes 2026-02-19T20:46:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 215241728 bytes 2026-02-19T20:46:15.082 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:15.078+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:46:16.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 305574912 bytes 2026-02-19T20:46:16.944 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:46:16.938+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aab60 2026-02-19T20:46:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 320276480 bytes 2026-02-19T20:46:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 333846528 bytes 2026-02-19T20:46:17.664 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:17.658+0000 7fe4dd68c640 -1 reset not still connected to 0x564a005f6ea0 2026-02-19T20:46:17.664 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:17.658+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00950000 2026-02-19T20:46:17.664 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:17.658+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3bba0 2026-02-19T20:46:17.664 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:17.658+0000 7fe4dd68c640 -1 reset not still connected to 0x564a08249a00 2026-02-19T20:46:17.664 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:17.658+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2cc30 2026-02-19T20:46:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 350998528 bytes 2026-02-19T20:46:18.103 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:46:18.098+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aab60 2026-02-19T20:46:18.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 388306944 bytes 2026-02-19T20:46:18.875 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:46:18.870+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:46:19.109 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:19.102+0000 7fe4dd68c640 -1 reset not still connected to 0x564a005f6ea0 2026-02-19T20:46:19.109 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:19.102+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00950000 2026-02-19T20:46:19.111 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:19.106+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3bba0 2026-02-19T20:46:19.111 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:19.106+0000 7fe4dd68c640 -1 reset not still connected to 0x564a08249a00 2026-02-19T20:46:19.111 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:19.106+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2cc30 2026-02-19T20:46:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 472209408 bytes 2026-02-19T20:46:19.892 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:46:19.886+0000 7fd63ae12640 -1 reset not still connected to 0x55f110e16270 2026-02-19T20:46:19.892 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:46:19.886+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:46:19.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 490288128 bytes 2026-02-19T20:46:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 494738432 bytes 2026-02-19T20:46:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:46:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:46:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:46:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:46:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:46:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_19 2026-02-19T20:46:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 journal 128M 2026-02-19T20:46:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_19 128M 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:46:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:46:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:46:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:46:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_19 2026-02-19T20:46:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:46:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:46:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:46:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:46:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:46:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_19 2026-02-19T20:46:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:46:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:46:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=8 2026-02-19T20:46:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:46:20.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:46:21.830 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:21.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2cc30 2026-02-19T20:46:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 50480128 bytes 2026-02-19T20:46:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 77091840 bytes 2026-02-19T20:46:23.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 109224960 bytes 2026-02-19T20:46:24.372 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:24.366+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd930 2026-02-19T20:46:24.372 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:24.366+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:46:24.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 172752896 bytes 2026-02-19T20:46:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 223271936 bytes 2026-02-19T20:46:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 234502144 bytes 2026-02-19T20:46:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 238581760 bytes 2026-02-19T20:46:25.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 239188992 bytes 2026-02-19T20:46:26.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 277632000 bytes 2026-02-19T20:46:26.516 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7506468545642659) > weight (0.25) 2026-02-19T20:46:26.516 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9434652553658109) > weight (0.25) 2026-02-19T20:46:26.516 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8450039156315634) > weight (0.25) 2026-02-19T20:46:26.516 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-19T20:46:26.516 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-19T20:46:26.517 INFO:tasks.rbd_mirror_thrash:waiting for 8.900843814456685 secs before reviving daemons 2026-02-19T20:46:26.517 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T20:46:26.503+0000 7fda802ee640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 78976) UID: 1000 2026-02-19T20:46:26.518 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T20:46:26.507+0000 7fda78adf640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55d5e482f180 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-19T20:46:26.518 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T20:46:26.507+0000 7fda78adf640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55d5e482f180 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-19T20:46:26.900 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:26.894+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec3380 2026-02-19T20:46:26.900 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:26.894+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:46:26.900 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:26.894+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:46:26.900 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:26.894+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:46:26.900 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:26.894+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:46:26.901 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:26.894+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:46:26.901 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:26.894+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:46:27.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 319531008 bytes 2026-02-19T20:46:27.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 339700736 bytes 2026-02-19T20:46:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 354576384 bytes 2026-02-19T20:46:28.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 396241920 bytes 2026-02-19T20:46:28.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_20 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 journal 128M 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_20 128M 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:46:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:28.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:46:28.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:46:28.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:46:28.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:28.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_20 2026-02-19T20:46:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:46:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:46:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:46:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:46:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:46:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_20 2026-02-19T20:46:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:46:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:46:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=6 2026-02-19T20:46:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:46:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:46:29.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 52442112 bytes 2026-02-19T20:46:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 121695232 bytes 2026-02-19T20:46:31.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 156325888 bytes 2026-02-19T20:46:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 198686720 bytes 2026-02-19T20:46:32.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 202271744 bytes 2026-02-19T20:46:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 251793408 bytes 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_21 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 journal 128M 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_21 128M 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:46:34.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:34.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:46:34.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:46:34.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:46:34.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:34.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_21 2026-02-19T20:46:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:46:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:46:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:46:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:46:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:46:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_21 2026-02-19T20:46:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:46:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:46:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=8 2026-02-19T20:46:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:46:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:46:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 21677056 bytes 2026-02-19T20:46:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 25743360 bytes 2026-02-19T20:46:35.418 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-19T20:46:35.419 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:46:35.419 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-19T20:46:35.419 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-19T20:46:35.419 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-19T20:46:35.421 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-19T20:46:35.422 INFO:tasks.rbd_mirror_thrash:waiting for 101.91625228201232 secs before thrashing 2026-02-19T20:46:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 97205248 bytes 2026-02-19T20:46:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 151490560 bytes 2026-02-19T20:46:38.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 206350336 bytes 2026-02-19T20:46:40.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 274174976 bytes 2026-02-19T20:46:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 329617408 bytes 2026-02-19T20:46:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 390694912 bytes 2026-02-19T20:46:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:46:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:46:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:46:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:46:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:46:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_22 2026-02-19T20:46:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 journal 128M 2026-02-19T20:46:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_22 128M 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:46:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:46:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:46:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_22 2026-02-19T20:46:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:46:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:46:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:46:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:46:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:46:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_22 2026-02-19T20:46:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:46:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:46:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=10 2026-02-19T20:46:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:46:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:46:42.828 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:46:42.822+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:46:43.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 49730560 bytes 2026-02-19T20:46:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 86622208 bytes 2026-02-19T20:46:44.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 111223808 bytes 2026-02-19T20:46:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 196235264 bytes 2026-02-19T20:46:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 251718656 bytes 2026-02-19T20:46:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 278693888 bytes 2026-02-19T20:46:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 280276992 bytes 2026-02-19T20:46:47.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 349543424 bytes 2026-02-19T20:46:48.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 408868864 bytes 2026-02-19T20:46:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 466758656 bytes 2026-02-19T20:46:50.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 505757696 bytes 2026-02-19T20:46:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 581833728 bytes 2026-02-19T20:46:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_23 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 journal 128M 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_23 128M 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:46:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_23 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_23 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:46:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:46:52.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=10 2026-02-19T20:46:52.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:46:52.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:46:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 601088 bytes 2026-02-19T20:46:52.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 1218560 bytes 2026-02-19T20:46:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 38212608 bytes 2026-02-19T20:46:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 93428736 bytes 2026-02-19T20:46:55.916 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:46:55.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb3790 2026-02-19T20:46:55.916 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:46:55.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e4483b860 2026-02-19T20:46:55.916 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:46:55.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ed00 2026-02-19T20:46:55.916 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:46:55.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:46:55.917 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:46:55.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:46:55.917 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:46:55.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:46:55.917 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:46:55.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:46:56.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 178147328 bytes 2026-02-19T20:46:57.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 228337664 bytes 2026-02-19T20:46:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 288812032 bytes 2026-02-19T20:46:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 332694528 bytes 2026-02-19T20:46:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 390774784 bytes 2026-02-19T20:47:00.393 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:00.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e31d37450 2026-02-19T20:47:00.393 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:00.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e324ee9c0 2026-02-19T20:47:00.393 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:00.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:47:00.393 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:00.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719790 2026-02-19T20:47:00.394 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:00.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e8f0 2026-02-19T20:47:00.394 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:00.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaf70 2026-02-19T20:47:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 451039232 bytes 2026-02-19T20:47:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 466029568 bytes 2026-02-19T20:47:01.901 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4408048117962171) > weight (0.25) 2026-02-19T20:47:01.901 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-19T20:47:01.902 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-19T20:47:01.902 INFO:tasks.rbd_mirror_thrash:waiting for 7.35984440430041 secs before reviving daemons 2026-02-19T20:47:01.966 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T20:47:01.891+0000 7fac51578640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 30639) UID: 1000 2026-02-19T20:47:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 537431040 bytes 2026-02-19T20:47:02.038 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:47:02.027+0000 7f5ce3652640 -1 reset not still connected to 0x5610321c9a00 2026-02-19T20:47:02.486 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:02.482+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:02.487 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:02.482+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:47:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_24 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 journal 128M 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_24 128M 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:47:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:47:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_24 2026-02-19T20:47:02.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:47:02.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:02.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:47:02.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:47:02.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:47:02.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:47:02.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_24 2026-02-19T20:47:02.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:02.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:02.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:47:02.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:47:02.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=7 2026-02-19T20:47:02.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:47:02.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:47:03.426 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:03.422+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:03.427 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:03.422+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:03.607 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:03.602+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3a000 2026-02-19T20:47:03.609 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:03.602+0000 7fe4dd68c640 -1 reset not still connected to 0x564a07d6b520 2026-02-19T20:47:03.609 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:03.602+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd930 2026-02-19T20:47:03.609 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:03.602+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:04.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 72407040 bytes 2026-02-19T20:47:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 92063744 bytes 2026-02-19T20:47:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 94433280 bytes 2026-02-19T20:47:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 176502784 bytes 2026-02-19T20:47:05.967 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:05.962+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:05.967 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:05.962+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:05.967 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:05.962+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:05.968 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:05.962+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:05.968 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:05.962+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:06.369 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:06.362+0000 7fd63ae12640 -1 reset not still connected to 0x55f110e16270 2026-02-19T20:47:06.369 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:06.362+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:47:06.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 217886720 bytes 2026-02-19T20:47:06.734 INFO:tasks.ceph.cluster2.osd.1.trial151.stderr:2026-02-19T20:47:06.723+0000 7f58ec161640 -1 reset not still connected to 0x5561934ec4e0 2026-02-19T20:47:06.783 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:47:06.771+0000 7f5ce3652640 -1 reset not still connected to 0x5610608996c0 2026-02-19T20:47:06.783 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:47:06.771+0000 7f5ce3652640 -1 reset not still connected to 0x56106257c8f0 2026-02-19T20:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 276472832 bytes 2026-02-19T20:47:08.239 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:08.234+0000 7fd63ae12640 -1 reset not still connected to 0x55f11003bc70 2026-02-19T20:47:08.240 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:08.234+0000 7fd63ae12640 -1 reset not still connected to 0x55f11025e820 2026-02-19T20:47:08.240 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:08.234+0000 7fd63ae12640 -1 reset not still connected to 0x55f110e17ba0 2026-02-19T20:47:08.240 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:08.234+0000 7fd63ae12640 -1 reset not still connected to 0x55f111fd0a90 2026-02-19T20:47:08.240 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:08.234+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74a90 2026-02-19T20:47:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 317284352 bytes 2026-02-19T20:47:08.687 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:08.682+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:47:08.687 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:08.682+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:08.687 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:08.682+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:47:08.687 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:08.682+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:08.829 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:08.822+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:08.830 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:08.822+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:47:08.830 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:08.822+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:08.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 339743744 bytes 2026-02-19T20:47:09.258 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:47:09.247+0000 7f5ce3652640 -1 reset not still connected to 0x56102b916f70 2026-02-19T20:47:09.259 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:47:09.247+0000 7f5ce3652640 -1 reset not still connected to 0x56104f0dbd40 2026-02-19T20:47:09.259 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:47:09.247+0000 7f5ce3652640 -1 reset not still connected to 0x5610608996c0 2026-02-19T20:47:09.259 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:47:09.247+0000 7f5ce3652640 -1 reset not still connected to 0x56106257c8f0 2026-02-19T20:47:09.263 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-19T20:47:09.263 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:47:09.264 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-19T20:47:09.264 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-19T20:47:09.264 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-19T20:47:09.266 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-19T20:47:09.266 INFO:tasks.rbd_mirror_thrash:waiting for 107.09486387234526 secs before thrashing 2026-02-19T20:47:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:47:09.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:47:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:47:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:47:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:47:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_25 2026-02-19T20:47:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 journal 128M 2026-02-19T20:47:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_25 128M 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:47:09.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:09.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:47:09.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:47:09.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:47:09.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:09.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_25 2026-02-19T20:47:09.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:47:09.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:09.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:47:09.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:47:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:47:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:47:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_25 2026-02-19T20:47:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:47:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:47:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=14 2026-02-19T20:47:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:47:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:47:10.203 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:10.198+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:10.204 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:10.198+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd5ba0 2026-02-19T20:47:10.205 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:10.198+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:10.304 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:10.298+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b77d40 2026-02-19T20:47:10.304 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:10.298+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d99110 2026-02-19T20:47:10.305 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:10.298+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f01fee0 2026-02-19T20:47:10.305 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:10.298+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78b520 2026-02-19T20:47:10.305 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:10.298+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaea0 2026-02-19T20:47:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 30900224 bytes 2026-02-19T20:47:10.863 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:10.858+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b77d40 2026-02-19T20:47:10.864 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:10.858+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d99110 2026-02-19T20:47:10.864 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:10.858+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f01fee0 2026-02-19T20:47:10.864 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:10.858+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78b520 2026-02-19T20:47:10.864 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:10.858+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaea0 2026-02-19T20:47:11.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 66385920 bytes 2026-02-19T20:47:11.255 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.250+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:47:11.255 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.250+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:11.255 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.250+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:11.255 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.250+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:11.256 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.250+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:11.256 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.250+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:11.256 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.250+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:11.299 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.294+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:47:11.300 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.294+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:11.300 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.294+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:11.301 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.294+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:11.301 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.294+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:11.301 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.294+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:11.301 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.294+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:11.391 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:47:11.392 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:11.392 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:11.392 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:11.393 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:11.393 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:11.393 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:11.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 117206016 bytes 2026-02-19T20:47:12.358 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:12.354+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:47:12.358 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:12.354+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:12.358 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:12.354+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:12.359 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:12.354+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:12.359 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:12.354+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:12.360 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:12.354+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:12.360 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:12.354+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:13.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 182702080 bytes 2026-02-19T20:47:13.618 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:13.610+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:14.015 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.010+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:14.015 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.010+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:14.016 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.010+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:14.016 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.010+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:14.017 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.010+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 203885568 bytes 2026-02-19T20:47:14.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 204486656 bytes 2026-02-19T20:47:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 211112960 bytes 2026-02-19T20:47:14.321 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.314+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:47:14.321 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.314+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:14.321 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.314+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:14.322 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:14.322 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:14.322 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:14.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:15.113 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:15.106+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd930 2026-02-19T20:47:15.113 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:15.106+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:15.200 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:15.190+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:15.200 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:15.194+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:15.200 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:15.194+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:15.200 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:15.194+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:15.201 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:15.194+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 295312384 bytes 2026-02-19T20:47:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 351264768 bytes 2026-02-19T20:47:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 399159296 bytes 2026-02-19T20:47:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 417789952 bytes 2026-02-19T20:47:17.329 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:17.322+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:17.833 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:17.826+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:17.833 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:17.826+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:17.834 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:17.826+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:17.834 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:17.826+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:17.834 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:17.826+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 472339456 bytes 2026-02-19T20:47:18.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 518030336 bytes 2026-02-19T20:47:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 535397376 bytes 2026-02-19T20:47:19.049 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:19.042+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:47:19.049 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:19.042+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:47:19.049 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:19.042+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:19.050 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:19.046+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:47:19.050 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:19.046+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:19.981 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:19.974+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 600528896 bytes 2026-02-19T20:47:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 603030528 bytes 2026-02-19T20:47:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 634029056 bytes 2026-02-19T20:47:20.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T20:47:20.731+0000 7f9ff06f3640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x560516866a80 finish: resending after timeout 2026-02-19T20:47:20.830 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:20.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 634632192 bytes 2026-02-19T20:47:21.107 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:21.102+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8c1a0 2026-02-19T20:47:21.107 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:21.102+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3a000 2026-02-19T20:47:21.107 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:21.102+0000 7fe4dd68c640 -1 reset not still connected to 0x564a07d6b520 2026-02-19T20:47:21.108 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:21.102+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd930 2026-02-19T20:47:21.108 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:21.102+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:21.392 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:21.386+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaea0 2026-02-19T20:47:21.491 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:21.486+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:47:21.492 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:21.486+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:21.492 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:21.486+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:47:21.492 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:21.486+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:21.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 657773568 bytes 2026-02-19T20:47:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2000 IOs, 700081152 bytes 2026-02-19T20:47:22.819 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:22.814+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:47:22.822 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:22.814+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:47:22.822 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:22.814+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:47:22.822 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:22.814+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:47:22.822 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:22.814+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:47:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2100 IOs, 725429248 bytes 2026-02-19T20:47:23.570 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:23.566+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719790 2026-02-19T20:47:23.570 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:23.566+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e8f0 2026-02-19T20:47:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_26 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 journal 128M 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_26 128M 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_26 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_26 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:47:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:47:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=11 2026-02-19T20:47:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:47:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:47:24.292 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:24.286+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:24.292 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:24.286+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:24.292 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:24.286+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:24.342 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.338+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:47:24.342 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.338+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:47:24.342 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.338+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:47:24.342 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.338+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:47:24.343 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.338+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29a00 2026-02-19T20:47:24.343 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.338+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:47:24.343 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.338+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:47:24.343 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.338+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34a90 2026-02-19T20:47:24.602 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.594+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:47:24.602 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.598+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:47:24.604 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.598+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:47:24.604 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.598+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:47:24.605 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.598+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29a00 2026-02-19T20:47:24.605 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.598+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:47:24.605 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.598+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:47:24.605 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:24.598+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34a90 2026-02-19T20:47:24.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 46131200 bytes 2026-02-19T20:47:25.098 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:25.094+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4820 2026-02-19T20:47:25.098 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:25.094+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:47:25.099 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:25.094+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:47:25.099 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:25.094+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:25.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 83680256 bytes 2026-02-19T20:47:25.482 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:25.478+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:27.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 162772992 bytes 2026-02-19T20:47:27.643 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.638+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:27.644 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.638+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:27.644 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.638+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:27.644 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.638+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:27.647 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.642+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:47:27.648 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.642+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:27.648 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.642+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:27.648 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.642+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:27.649 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.642+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:27.649 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.642+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:27.649 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.642+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:27.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.906+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:47:27.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.906+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:27.914 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.906+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:27.914 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.906+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:27.916 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:27.916 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:27.916 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:27.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:28.219 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44c30 2026-02-19T20:47:28.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e336b8750 2026-02-19T20:47:28.222 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb2a90 2026-02-19T20:47:28.222 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb3790 2026-02-19T20:47:28.222 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e4483b860 2026-02-19T20:47:28.222 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ed00 2026-02-19T20:47:28.222 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fa00 2026-02-19T20:47:28.222 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:47:28.223 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:47:28.223 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:47:28.223 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:47:28.223 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:28.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:47:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 199474176 bytes 2026-02-19T20:47:28.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 215497728 bytes 2026-02-19T20:47:29.004 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:28.998+0000 7fd63ae12640 -1 reset not still connected to 0x55f110e16270 2026-02-19T20:47:29.004 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:28.998+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:47:29.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 303848448 bytes 2026-02-19T20:47:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 332133376 bytes 2026-02-19T20:47:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 332750848 bytes 2026-02-19T20:47:30.609 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:30.601+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:47:30.609 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:30.601+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:47:30.609 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:30.601+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29a00 2026-02-19T20:47:30.610 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:30.605+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:47:30.611 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:30.605+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:47:30.611 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:30.605+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34a90 2026-02-19T20:47:30.870 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:30.861+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:47:31.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 409970688 bytes 2026-02-19T20:47:31.687 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:31.681+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:31.721 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:31.713+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f01fee0 2026-02-19T20:47:31.721 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:31.713+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78b520 2026-02-19T20:47:31.721 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:31.713+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f96c0 2026-02-19T20:47:31.721 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:31.713+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaea0 2026-02-19T20:47:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 438475776 bytes 2026-02-19T20:47:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 476874752 bytes 2026-02-19T20:47:32.293 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:32.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:47:32.293 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:32.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:47:32.293 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:32.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34a90 2026-02-19T20:47:32.404 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:32.397+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:32.406 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:32.401+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:32.529 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:32.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:47:32.533 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:32.525+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:47:32.533 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:32.525+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:47:32.535 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:32.529+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:47:32.535 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:32.529+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa270 2026-02-19T20:47:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 512843776 bytes 2026-02-19T20:47:33.224 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:33.217+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:33.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 534225920 bytes 2026-02-19T20:47:33.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 547861504 bytes 2026-02-19T20:47:33.780 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:33.773+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:33.780 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:33.773+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:47:33.781 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:33.773+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:33.798 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:33.793+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:47:33.798 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:33.793+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:47:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 569410560 bytes 2026-02-19T20:47:34.023 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:34.017+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:47:34.023 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:34.017+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:34.025 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:34.017+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:47:34.025 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:34.017+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 575447040 bytes 2026-02-19T20:47:34.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 589456384 bytes 2026-02-19T20:47:34.419 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:34.413+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:34.419 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:34.413+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:47:34.419 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:34.413+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 617102336 bytes 2026-02-19T20:47:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:47:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_27 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 journal 128M 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:47:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_27 128M 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:47:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:47:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_27 2026-02-19T20:47:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:47:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:47:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:47:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:47:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:47:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_27 2026-02-19T20:47:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:47:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:47:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=5 2026-02-19T20:47:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:47:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:47:36.308 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:36.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 51058688 bytes 2026-02-19T20:47:36.513 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:36.505+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:47:36.513 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:36.505+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:36.513 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:36.505+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:36.514 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:36.505+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:36.714 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:36.709+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa270 2026-02-19T20:47:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 92855296 bytes 2026-02-19T20:47:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 155515904 bytes 2026-02-19T20:47:38.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 192322560 bytes 2026-02-19T20:47:39.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 239881216 bytes 2026-02-19T20:47:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 292000768 bytes 2026-02-19T20:47:39.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:47:39.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:47:39.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:47:39.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:47:39.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:47:39.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_28 2026-02-19T20:47:39.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 journal 128M 2026-02-19T20:47:39.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_28 128M 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:47:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:47:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:47:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:47:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_28 2026-02-19T20:47:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:47:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:47:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:47:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:47:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:47:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_28 2026-02-19T20:47:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:47:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:47:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=10 2026-02-19T20:47:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:47:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:47:40.636 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:40.629+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:40.952 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:40.945+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec3380 2026-02-19T20:47:40.952 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:40.945+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d6c0 2026-02-19T20:47:40.952 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:40.945+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:47:40.952 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:40.945+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:47:40.953 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:40.945+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:47:40.953 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:40.945+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:47:40.953 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:40.945+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:47:40.953 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:40.945+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:47:41.420 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:41.413+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:41.420 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:41.413+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:41.421 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:41.413+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:41.421 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:41.413+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:41.422 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:41.413+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 65331200 bytes 2026-02-19T20:47:41.675 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:41.669+0000 7fe4dd68c640 -1 reset not still connected to 0x564a07d6b520 2026-02-19T20:47:41.676 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:41.669+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0b3100d0 2026-02-19T20:47:41.676 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:41.669+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0820 2026-02-19T20:47:41.676 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:41.669+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:41.921 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:41.913+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:47:41.921 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:41.913+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719790 2026-02-19T20:47:41.921 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:41.913+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e8f0 2026-02-19T20:47:41.921 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:41.913+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaf70 2026-02-19T20:47:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 107304960 bytes 2026-02-19T20:47:42.834 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:42.825+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719790 2026-02-19T20:47:42.834 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:42.825+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e8f0 2026-02-19T20:47:42.834 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:42.825+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaf70 2026-02-19T20:47:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 199881728 bytes 2026-02-19T20:47:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 210894848 bytes 2026-02-19T20:47:43.916 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.909+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:47:43.917 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.909+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:43.918 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.909+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:43.918 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.909+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:43.918 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.913+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:43.919 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.913+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:43.919 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.913+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:43.958 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.953+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:43.958 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.953+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:43.960 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.953+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:43.961 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.953+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:43.961 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:43.953+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:44.050 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:44.041+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4270 2026-02-19T20:47:44.050 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:44.041+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:47:44.050 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:44.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:47:44.051 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:44.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:47:44.051 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:44.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:47:44.051 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:44.045+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29a00 2026-02-19T20:47:44.051 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:44.045+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:47:44.051 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:44.045+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:47:44.052 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:44.045+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34a90 2026-02-19T20:47:44.224 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:44.217+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:44.388 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:44.381+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:44.389 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:44.381+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:44.389 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:44.381+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 268138496 bytes 2026-02-19T20:47:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 282133504 bytes 2026-02-19T20:47:45.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 307549184 bytes 2026-02-19T20:47:45.796 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:45.789+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 335557632 bytes 2026-02-19T20:47:45.974 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:45.969+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd930 2026-02-19T20:47:45.975 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:45.969+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0b3100d0 2026-02-19T20:47:45.975 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:45.969+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0820 2026-02-19T20:47:45.975 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:45.969+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:46.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 367476736 bytes 2026-02-19T20:47:46.547 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:46.541+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0820 2026-02-19T20:47:46.547 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:46.541+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:47.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 408625152 bytes 2026-02-19T20:47:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 421798912 bytes 2026-02-19T20:47:48.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 439024640 bytes 2026-02-19T20:47:48.432 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.425+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb3790 2026-02-19T20:47:48.432 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.425+0000 7f93a8cad640 -1 reset not still connected to 0x556e4483b860 2026-02-19T20:47:48.432 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.425+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ed00 2026-02-19T20:47:48.432 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.425+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fa00 2026-02-19T20:47:48.432 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.425+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:47:48.434 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.429+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:47:48.434 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.429+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:47:48.434 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.429+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:47:48.434 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.429+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:47:48.435 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.429+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa270 2026-02-19T20:47:48.973 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.965+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:48.974 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.965+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:48.975 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:48.969+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 506318848 bytes 2026-02-19T20:47:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:47:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:47:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:47:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:47:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_29 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 journal 128M 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_29 128M 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:47:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_29 2026-02-19T20:47:49.943 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:49.933+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:49.943 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:49.933+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:47:49.943 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:49.933+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:47:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:47:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:47:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:47:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:47:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_29 2026-02-19T20:47:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:47:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:47:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:47:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:47:49.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=12 2026-02-19T20:47:49.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:47:49.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:47:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 62117888 bytes 2026-02-19T20:47:51.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 65197056 bytes 2026-02-19T20:47:51.324 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:51.317+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:47:51.324 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:51.317+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:47:51.324 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:51.317+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:51.325 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:51.317+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:51.326 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:51.317+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:51.326 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:51.317+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:51.365 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:51.357+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:51.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 103610368 bytes 2026-02-19T20:47:52.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 120861696 bytes 2026-02-19T20:47:52.779 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:52.773+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f96c0 2026-02-19T20:47:52.779 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:52.773+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaea0 2026-02-19T20:47:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 146328576 bytes 2026-02-19T20:47:53.120 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.113+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:53.120 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.113+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:53.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.113+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:53.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.113+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:53.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.113+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:53.181 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.173+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:47:53.181 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.173+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:47:53.183 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.177+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:47:53.183 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.177+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:47:53.183 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.177+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:53.381 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:53.373+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:54.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 210590720 bytes 2026-02-19T20:47:54.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 269808640 bytes 2026-02-19T20:47:55.116 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:55.109+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd930 2026-02-19T20:47:55.117 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:55.109+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0b3100d0 2026-02-19T20:47:55.117 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:55.109+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0820 2026-02-19T20:47:55.117 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:55.109+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:47:55.257 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:55.249+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:55.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 304171008 bytes 2026-02-19T20:47:56.307 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.301+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:47:56.308 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.301+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc5b0 2026-02-19T20:47:56.308 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.301+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:56.309 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.301+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:47:56.309 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.301+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:56.311 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.305+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4820 2026-02-19T20:47:56.312 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.305+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:47:56.313 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.305+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:47:56.313 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.305+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc5b0 2026-02-19T20:47:56.313 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.305+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:47:56.313 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.305+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:47:56.313 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:56.305+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:47:56.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 337107968 bytes 2026-02-19T20:47:57.454 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:57.445+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78b520 2026-02-19T20:47:57.454 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:57.445+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f96c0 2026-02-19T20:47:57.454 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:57.445+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaea0 2026-02-19T20:47:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 393039872 bytes 2026-02-19T20:47:58.022 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:58.013+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:47:58.218 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:58.209+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74340 2026-02-19T20:47:58.289 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:58.281+0000 7f93a8cad640 -1 reset not still connected to 0x556e31d37450 2026-02-19T20:47:58.290 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:58.281+0000 7f93a8cad640 -1 reset not still connected to 0x556e324ee9c0 2026-02-19T20:47:58.290 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:58.281+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:47:58.291 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:58.281+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719790 2026-02-19T20:47:58.291 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:58.281+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e8f0 2026-02-19T20:47:58.291 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:47:58.285+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaf70 2026-02-19T20:47:58.330 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:58.321+0000 7fd63ae12640 -1 reset not still connected to 0x55f11003bc70 2026-02-19T20:47:58.330 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:58.321+0000 7fd63ae12640 -1 reset not still connected to 0x55f110e17ba0 2026-02-19T20:47:58.330 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:58.321+0000 7fd63ae12640 -1 reset not still connected to 0x55f111fd0a90 2026-02-19T20:47:58.330 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:47:58.321+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74a90 2026-02-19T20:47:58.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 436978688 bytes 2026-02-19T20:47:58.894 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:58.889+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d6c0 2026-02-19T20:47:58.895 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:58.889+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:47:58.895 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:58.889+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:47:58.896 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:58.889+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:47:58.896 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:58.889+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:47:58.896 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:58.889+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:47:58.896 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:58.889+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:47:59.035 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:59.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d6c0 2026-02-19T20:47:59.036 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:59.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:47:59.036 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:59.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:47:59.036 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:59.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:47:59.037 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:59.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:47:59.037 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:59.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:47:59.037 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:47:59.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:47:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 469901312 bytes 2026-02-19T20:48:00.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 511924224 bytes 2026-02-19T20:48:00.327 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:00.321+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:48:00.329 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:00.321+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:48:00.329 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:00.321+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc5b0 2026-02-19T20:48:00.329 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:00.321+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:48:00.329 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:00.321+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:00.329 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:00.321+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 548039680 bytes 2026-02-19T20:48:00.838 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:00.833+0000 7f93a8cad640 -1 reset not still connected to 0x556e4483b860 2026-02-19T20:48:00.839 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:00.833+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ed00 2026-02-19T20:48:00.841 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:00.833+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fa00 2026-02-19T20:48:00.841 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:00.833+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:48:00.841 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:00.833+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:48:00.841 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:00.833+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:48:00.841 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:00.833+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:48:00.841 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:00.833+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:48:00.841 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:00.833+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa270 2026-02-19T20:48:01.264 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:01.257+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:01.264 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:01.257+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:48:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 590714880 bytes 2026-02-19T20:48:01.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 638155776 bytes 2026-02-19T20:48:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 638767104 bytes 2026-02-19T20:48:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:48:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:48:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:48:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:48:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_30 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 journal 128M 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:48:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_30 128M 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_30 2026-02-19T20:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_30 2026-02-19T20:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:48:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=9 2026-02-19T20:48:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:48:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:48:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 65372160 bytes 2026-02-19T20:48:02.920 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.913+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d6c0 2026-02-19T20:48:02.920 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.913+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:48:02.921 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.913+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:48:02.921 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.913+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:48:02.922 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.917+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:48:02.923 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.917+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:48:02.945 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.937+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:48:02.946 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.941+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:48:02.947 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.941+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:48:02.948 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.941+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:48:02.948 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.941+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:48:02.949 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:02.941+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:48:03.519 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:03.513+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d6c0 2026-02-19T20:48:03.520 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:03.513+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042f70 2026-02-19T20:48:03.521 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:03.513+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043e10 2026-02-19T20:48:03.521 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:03.513+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29930 2026-02-19T20:48:03.522 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:03.513+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f22340 2026-02-19T20:48:03.522 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:03.513+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f232b0 2026-02-19T20:48:03.522 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:03.513+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23380 2026-02-19T20:48:03.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 112588800 bytes 2026-02-19T20:48:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 113197056 bytes 2026-02-19T20:48:04.026 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:04.017+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:04.029 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:04.021+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:48:04.029 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:04.021+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:48:04.029 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:04.021+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:04.029 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:04.021+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:04.206 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:04.197+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74340 2026-02-19T20:48:04.206 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:04.197+0000 7fd63ae12640 -1 reset not still connected to 0x55f134921c70 2026-02-19T20:48:04.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 150776832 bytes 2026-02-19T20:48:04.594 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:04.585+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdcea0 2026-02-19T20:48:04.595 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:04.589+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:04.595 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:04.589+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:04.707 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:04.701+0000 7f93a8cad640 -1 reset not still connected to 0x556e324ee9c0 2026-02-19T20:48:04.707 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:04.701+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:48:04.708 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:04.701+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719790 2026-02-19T20:48:04.708 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:04.701+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e8f0 2026-02-19T20:48:04.708 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:04.701+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaf70 2026-02-19T20:48:05.695 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:05.689+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:48:05.697 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:05.689+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:05.697 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:05.689+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:05.697 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:05.689+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:48:05.697 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:05.689+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:48:05.697 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:05.689+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:48:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 182406144 bytes 2026-02-19T20:48:06.272 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb3790 2026-02-19T20:48:06.272 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e4483b860 2026-02-19T20:48:06.272 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ed00 2026-02-19T20:48:06.273 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fa00 2026-02-19T20:48:06.273 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:48:06.273 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:48:06.274 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:48:06.274 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:48:06.274 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e21a0 2026-02-19T20:48:06.274 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:48:06.274 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.265+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa270 2026-02-19T20:48:06.313 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.305+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:48:06.313 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.305+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:48:06.314 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.305+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:06.314 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.305+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:06.315 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.309+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:48:06.315 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.309+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:48:06.316 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.309+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:48:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 194305024 bytes 2026-02-19T20:48:06.363 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.357+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d99110 2026-02-19T20:48:06.363 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.357+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f01fee0 2026-02-19T20:48:06.363 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.357+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78b520 2026-02-19T20:48:06.363 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.357+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f96c0 2026-02-19T20:48:06.363 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:06.357+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaea0 2026-02-19T20:48:06.778 INFO:tasks.ceph.cluster2.osd.1.trial151.stderr:2026-02-19T20:48:06.767+0000 7f58ec161640 -1 reset not still connected to 0x556199ef2750 2026-02-19T20:48:06.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 250077184 bytes 2026-02-19T20:48:07.605 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:07.597+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74340 2026-02-19T20:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 286794752 bytes 2026-02-19T20:48:07.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 287422464 bytes 2026-02-19T20:48:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 345306112 bytes 2026-02-19T20:48:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 353268736 bytes 2026-02-19T20:48:08.937 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:08.929+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:48:08.938 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:08.933+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa270 2026-02-19T20:48:09.118 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:09.109+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:09.121 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:09.113+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:48:09.334 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:09.325+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaf70 2026-02-19T20:48:09.471 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:09.465+0000 7fd63ae12640 -1 reset not still connected to 0x55f134920000 2026-02-19T20:48:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 424345600 bytes 2026-02-19T20:48:09.753 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:09.745+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:48:09.753 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:09.745+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:48:09.754 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:09.745+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:48:09.754 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:09.749+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:48:09.755 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:09.749+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e21a0 2026-02-19T20:48:09.755 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:09.749+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:48:09.756 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:09.749+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa270 2026-02-19T20:48:09.789 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:09.781+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:48:09.790 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:09.781+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:48:09.790 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:09.781+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:48:09.790 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:09.781+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:48:09.793 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:09.785+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29a00 2026-02-19T20:48:09.793 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:09.785+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:48:09.793 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:09.785+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:09.793 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:09.785+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34a90 2026-02-19T20:48:10.011 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.005+0000 7f93a8cad640 -1 reset not still connected to 0x556e4483b860 2026-02-19T20:48:10.012 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.005+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ed00 2026-02-19T20:48:10.013 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.005+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fa00 2026-02-19T20:48:10.015 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.009+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:48:10.015 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.009+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:48:10.015 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.009+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:48:10.016 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.009+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:48:10.016 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.009+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e21a0 2026-02-19T20:48:10.016 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.009+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:48:10.016 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.009+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa270 2026-02-19T20:48:10.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 466184192 bytes 2026-02-19T20:48:10.745 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e4483b860 2026-02-19T20:48:10.745 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ed00 2026-02-19T20:48:10.745 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fa00 2026-02-19T20:48:10.746 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:48:10.746 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:48:10.746 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:48:10.746 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:48:10.746 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e21a0 2026-02-19T20:48:10.747 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.741+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:48:10.747 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:10.741+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa270 2026-02-19T20:48:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:48:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:48:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_31 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 journal 128M 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_31 128M 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:48:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_31 2026-02-19T20:48:11.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:48:11.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:11.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:48:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:48:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:48:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:48:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_31 2026-02-19T20:48:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:48:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:48:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=5 2026-02-19T20:48:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:48:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:48:11.119 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:11.113+0000 7fd63ae12640 -1 reset not still connected to 0x55f10fc32410 2026-02-19T20:48:11.119 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:11.113+0000 7fd63ae12640 -1 reset not still connected to 0x55f10fc32750 2026-02-19T20:48:11.119 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:11.113+0000 7fd63ae12640 -1 reset not still connected to 0x55f10fec6750 2026-02-19T20:48:11.119 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:11.113+0000 7fd63ae12640 -1 reset not still connected to 0x55f110e16270 2026-02-19T20:48:11.119 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:11.113+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:48:11.518 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:11.509+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719790 2026-02-19T20:48:11.520 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:11.513+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e8f0 2026-02-19T20:48:11.521 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:11.513+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaf70 2026-02-19T20:48:11.889 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:11.881+0000 7f93a8cad640 -1 reset not still connected to 0x556e324ee9c0 2026-02-19T20:48:11.889 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:11.881+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:48:11.889 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:11.881+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719790 2026-02-19T20:48:11.890 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:11.881+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e8f0 2026-02-19T20:48:11.890 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:11.885+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaf70 2026-02-19T20:48:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 55432192 bytes 2026-02-19T20:48:12.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 103957504 bytes 2026-02-19T20:48:12.835 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:12.829+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ed00 2026-02-19T20:48:12.835 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:12.829+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fa00 2026-02-19T20:48:12.835 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:12.829+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:48:12.835 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:12.829+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:48:12.836 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:12.829+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:48:12.836 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:12.829+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:48:12.836 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:12.829+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e21a0 2026-02-19T20:48:12.836 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:12.829+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:48:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 127012864 bytes 2026-02-19T20:48:12.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 164488192 bytes 2026-02-19T20:48:13.008 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:13.001+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ed00 2026-02-19T20:48:13.008 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:13.001+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fa00 2026-02-19T20:48:13.009 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:13.001+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fee0 2026-02-19T20:48:13.012 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:13.005+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd380 2026-02-19T20:48:13.012 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:13.005+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a340 2026-02-19T20:48:13.012 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:13.005+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78a4e0 2026-02-19T20:48:13.012 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:13.005+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e21a0 2026-02-19T20:48:13.013 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:13.005+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3ad0 2026-02-19T20:48:13.013 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:13.005+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa270 2026-02-19T20:48:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 190404608 bytes 2026-02-19T20:48:14.444 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:14.437+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2e0d0 2026-02-19T20:48:14.445 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:14.437+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fd5f0 2026-02-19T20:48:14.445 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:14.437+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:14.445 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:14.437+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:14.446 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:14.441+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2000 2026-02-19T20:48:14.446 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:14.441+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:48:14.447 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:14.441+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f8ea0 2026-02-19T20:48:15.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 245564416 bytes 2026-02-19T20:48:15.765 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:15.757+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:48:15.765 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:15.757+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:48:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:48:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:48:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:48:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_32 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 journal 128M 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_32 128M 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_32 2026-02-19T20:48:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:48:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:48:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:48:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:48:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:48:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_32 2026-02-19T20:48:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:48:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:48:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=14 2026-02-19T20:48:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:48:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:48:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 54707200 bytes 2026-02-19T20:48:17.338 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T20:48:17.339 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T20:48:17.340 INFO:tasks.rbd_mirror_thrash:waiting for 3.2923079626675467 secs before reviving daemons 2026-02-19T20:48:17.340 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T20:48:17.327+0000 7f9ff16f5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 69713) UID: 1000 2026-02-19T20:48:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 55317504 bytes 2026-02-19T20:48:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 94931968 bytes 2026-02-19T20:48:18.650 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:18.641+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4270 2026-02-19T20:48:18.651 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:18.645+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:48:18.651 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:18.645+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:48:18.654 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:18.649+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:48:18.655 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:18.649+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:48:18.655 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:18.649+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:48:18.655 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:18.649+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd29a00 2026-02-19T20:48:18.655 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:18.649+0000 7fe4dd68c640 -1 reset not still connected to 0x564a22138a90 2026-02-19T20:48:18.655 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:18.649+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:18.655 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:18.649+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34a90 2026-02-19T20:48:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 170636288 bytes 2026-02-19T20:48:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 203982848 bytes 2026-02-19T20:48:20.633 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T20:48:20.633 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:48:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 264426496 bytes 2026-02-19T20:48:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 288228352 bytes 2026-02-19T20:48:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 313636864 bytes 2026-02-19T20:48:22.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 327925760 bytes 2026-02-19T20:48:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 348724224 bytes 2026-02-19T20:48:23.399 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T20:48:23.387+0000 7ff4b14f5640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x563dc6653c00 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-19T20:48:23.399 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T20:48:23.387+0000 7ff4b14f5640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x563dc6653c00 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-19T20:48:23.400 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:48:23.387+0000 7efcbd1bd640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x560a7ff87180 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-19T20:48:23.400 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:48:23.387+0000 7efcbd1bd640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x560a7ff87180 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-19T20:48:23.400 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T20:48:23.387+0000 7faaa525c640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55fa789a6380 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-19T20:48:23.400 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T20:48:23.387+0000 7faaa525c640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55fa789a6380 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-19T20:48:23.711 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:48:23.699+0000 7f784a414640 -1 bdev(0x557edd010400 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 3 2026-02-19T20:48:23.713 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:48:23.703+0000 7f784e41c640 -1 bdev(0x557edd010400 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 4 2026-02-19T20:48:23.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 430113792 bytes 2026-02-19T20:48:23.926 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:23.917+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:48:23.926 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:23.921+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:48:23.927 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:23.921+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:48:23.927 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:23.921+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:48:23.927 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:23.921+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:48:24.814 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:24.809+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:24.815 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:24.809+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:25.820 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:25.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8c1a0 2026-02-19T20:48:25.820 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:25.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00d08340 2026-02-19T20:48:25.821 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:25.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0b3104e0 2026-02-19T20:48:25.821 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:25.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0d00 2026-02-19T20:48:25.821 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:25.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:48:25.821 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:25.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:48:25.821 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:25.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:48:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 481890304 bytes 2026-02-19T20:48:26.604 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:26.597+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:48:26.605 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:26.597+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:48:26.606 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:26.601+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:48:26.606 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:26.601+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:48:26.607 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:26.601+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:48:26.607 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:26.601+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:26.607 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:26.601+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:48:26.607 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:26.601+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:48:26.607 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:26.601+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:26.607 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:26.601+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:48:26.735 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T20:48:26.735 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T20:48:26.735 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T20:48:26.738 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T20:48:26.738 INFO:tasks.rbd_mirror_thrash:waiting for 68.46396682753549 secs before thrashing 2026-02-19T20:48:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 522888192 bytes 2026-02-19T20:48:27.181 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:27.173+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:48:27.182 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:27.173+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:48:27.182 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:27.173+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1da00 2026-02-19T20:48:27.182 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:27.173+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:48:27.182 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:27.173+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:48:27.182 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:27.173+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:27.183 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:27.173+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:48:27.183 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:27.173+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:27.183 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:27.173+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 559816704 bytes 2026-02-19T20:48:28.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 675349504 bytes 2026-02-19T20:48:29.319 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:29.313+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:48:29.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 703834112 bytes 2026-02-19T20:48:29.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 705581056 bytes 2026-02-19T20:48:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 724207616 bytes 2026-02-19T20:48:29.914 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:48:29.903+0000 7f5ce3652640 -1 reset not still connected to 0x5610367a4b60 2026-02-19T20:48:29.914 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:48:29.903+0000 7f5ce3652640 -1 reset not still connected to 0x561060899790 2026-02-19T20:48:30.034 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.025+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:48:30.035 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.025+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:48:30.035 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:48:30.035 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:48:30.035 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:48:30.035 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:48:30.035 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:48:30.035 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:48:30.036 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:48:30.037 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.029+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:30.047 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:48:30.047 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:48:30.047 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:48:30.047 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:30.048 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:48:30.048 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:30.048 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:48:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:48:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:48:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_33 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 journal 128M 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:48:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_33 128M 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:48:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:48:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_33 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_33 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:48:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:48:30.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=10 2026-02-19T20:48:30.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:48:30.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:48:30.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 22131712 bytes 2026-02-19T20:48:30.812 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.805+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:48:30.813 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:30.805+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:31.055 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:31.049+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:31.057 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:31.049+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:48:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 99803136 bytes 2026-02-19T20:48:32.107 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.101+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44dd0 2026-02-19T20:48:32.108 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.101+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719450 2026-02-19T20:48:32.108 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.101+0000 7f93a8cad640 -1 reset not still connected to 0x556e34a28dd0 2026-02-19T20:48:32.108 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.101+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2f790 2026-02-19T20:48:32.109 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.101+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d996c0 2026-02-19T20:48:32.109 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.101+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fca90 2026-02-19T20:48:32.109 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.101+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fdd40 2026-02-19T20:48:32.109 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.101+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396c30 2026-02-19T20:48:32.109 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.101+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:48:32.246 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.241+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:48:32.255 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.249+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:48:32.255 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.249+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:32.256 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.249+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:48:32.257 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.249+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:48:32.257 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.249+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:32.258 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:32.253+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:48:32.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 126989312 bytes 2026-02-19T20:48:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 177457152 bytes 2026-02-19T20:48:34.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 200195072 bytes 2026-02-19T20:48:34.239 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.233+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:48:34.240 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.233+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:48:34.240 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.233+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:34.240 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.233+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:48:34.240 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.233+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:34.240 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.233+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:34.645 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.637+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:48:34.647 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.641+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1da00 2026-02-19T20:48:34.647 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.641+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:48:34.647 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.641+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:48:34.647 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.641+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:34.647 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.641+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:48:34.648 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.641+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:34.648 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.641+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:34.722 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.717+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00d08340 2026-02-19T20:48:34.723 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.717+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0b3104e0 2026-02-19T20:48:34.724 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.717+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0d00 2026-02-19T20:48:34.724 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.717+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:48:34.724 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.717+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:48:34.724 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.717+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28c30 2026-02-19T20:48:34.724 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.717+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:48:34.864 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.857+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:48:34.866 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.857+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28c30 2026-02-19T20:48:34.866 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:34.857+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:48:35.045 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:35.037+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719450 2026-02-19T20:48:35.045 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:35.037+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb2750 2026-02-19T20:48:35.045 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:35.037+0000 7f93a8cad640 -1 reset not still connected to 0x556e34a28dd0 2026-02-19T20:48:35.046 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:35.037+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2f790 2026-02-19T20:48:35.046 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:35.037+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d996c0 2026-02-19T20:48:35.046 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:35.037+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fca90 2026-02-19T20:48:35.046 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:35.037+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fdd40 2026-02-19T20:48:35.047 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:35.037+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396c30 2026-02-19T20:48:35.047 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:35.037+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:48:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 268899328 bytes 2026-02-19T20:48:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 277475328 bytes 2026-02-19T20:48:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 297494528 bytes 2026-02-19T20:48:36.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 337385472 bytes 2026-02-19T20:48:36.781 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:36.773+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:36.781 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:36.773+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:37.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 369927168 bytes 2026-02-19T20:48:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 414908416 bytes 2026-02-19T20:48:38.237 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:38.229+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:38.239 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:38.233+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:48:38.744 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:38.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:38.744 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:38.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:48:38.746 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:38.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:48:38.746 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:38.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:38.746 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:38.737+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:48:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 483449856 bytes 2026-02-19T20:48:39.523 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:39.517+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d3301a0 2026-02-19T20:48:39.523 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:39.517+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d99110 2026-02-19T20:48:39.523 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:39.517+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d995f0 2026-02-19T20:48:39.523 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:39.517+0000 7f93a8cad640 -1 reset not still connected to 0x556e493de340 2026-02-19T20:48:39.524 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:39.517+0000 7f93a8cad640 -1 reset not still connected to 0x556e493dfad0 2026-02-19T20:48:39.524 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:39.517+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:48:39.524 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:39.517+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:48:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 549223424 bytes 2026-02-19T20:48:40.109 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:40.101+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00d08340 2026-02-19T20:48:40.110 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:40.101+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0b3104e0 2026-02-19T20:48:40.110 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:40.101+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0d00 2026-02-19T20:48:40.110 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:40.101+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:48:40.110 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:40.101+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:48:40.110 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:40.101+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28c30 2026-02-19T20:48:40.111 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:40.101+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:48:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:48:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:48:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_34 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 journal 128M 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:48:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_34 128M 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:48:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_34 2026-02-19T20:48:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:48:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:48:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:48:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:48:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:48:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_34 2026-02-19T20:48:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:48:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:48:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=12 2026-02-19T20:48:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:48:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:48:40.434 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:40.424+0000 7fd63ae12640 -1 reset not still connected to 0x55f123850a90 2026-02-19T20:48:40.434 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:40.428+0000 7fd63ae12640 -1 reset not still connected to 0x55f13ef47d40 2026-02-19T20:48:40.959 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:40.952+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:48:40.960 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:40.952+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:48:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 60758016 bytes 2026-02-19T20:48:41.209 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:41.200+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74a90 2026-02-19T20:48:41.209 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:41.200+0000 7fd63ae12640 -1 reset not still connected to 0x55f123850a90 2026-02-19T20:48:41.210 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:41.200+0000 7fd63ae12640 -1 reset not still connected to 0x55f13ef47d40 2026-02-19T20:48:41.255 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:41.248+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74a90 2026-02-19T20:48:41.255 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:41.248+0000 7fd63ae12640 -1 reset not still connected to 0x55f123850a90 2026-02-19T20:48:41.256 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:41.248+0000 7fd63ae12640 -1 reset not still connected to 0x55f13ef47d40 2026-02-19T20:48:41.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 69141504 bytes 2026-02-19T20:48:41.340 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:41.332+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:41.341 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:41.332+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:48:41.344 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:41.336+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:48:41.344 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:41.336+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:41.344 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:41.336+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:48:41.585 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:41.576+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:48:41.585 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:41.576+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:41.586 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:41.576+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:48:41.586 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:41.576+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:41.586 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:41.576+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 145264640 bytes 2026-02-19T20:48:43.008 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:43.000+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:48:43.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 171857920 bytes 2026-02-19T20:48:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 230104064 bytes 2026-02-19T20:48:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 243308544 bytes 2026-02-19T20:48:44.735 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:44.728+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:48:44.735 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:44.728+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:48:44.735 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:44.728+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:48:44.735 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:44.728+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:48:44.735 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:44.728+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:48:44.735 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:44.728+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:48:44.735 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:44.728+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:48:44.735 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:44.728+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 258910208 bytes 2026-02-19T20:48:45.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 304733184 bytes 2026-02-19T20:48:46.040 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:46.032+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:48:46.041 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:46.032+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:48:46.041 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:46.032+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:46.042 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:46.032+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:48:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 408238080 bytes 2026-02-19T20:48:47.523 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:47.516+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:48:47.524 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:47.516+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:48:47.525 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:47.516+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:48:47.525 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:47.516+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:48:47.525 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:47.516+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:48:47.526 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:47.516+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:47.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 435611648 bytes 2026-02-19T20:48:48.420 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.412+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:48:48.421 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.412+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:48:48.422 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.412+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:48:48.422 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.412+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:48:48.422 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.412+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:48:48.484 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.476+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:48:48.485 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.476+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:48:48.486 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.476+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:48:48.486 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.476+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:48:48.486 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.476+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:48:48.486 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.476+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:48:48.486 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.476+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:48:48.487 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.480+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:48.542 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.532+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:48:48.542 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.532+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1da00 2026-02-19T20:48:48.542 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.532+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:48:48.542 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.532+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:48:48.542 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.532+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:48.543 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.536+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:48:48.543 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.536+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:48.543 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.536+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:48.559 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.552+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:48:48.560 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.552+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28c30 2026-02-19T20:48:48.560 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.552+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:48:48.911 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:48.904+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:48.912 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:48.904+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:48:48.912 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:48.904+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:48.912 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:48.904+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:48:48.980 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.972+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:48:48.980 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.972+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:48:48.981 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.972+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:48:48.981 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.972+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:48.982 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.972+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:48:48.983 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.976+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:48.983 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:48.976+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 522175488 bytes 2026-02-19T20:48:49.384 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:49.376+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fca90 2026-02-19T20:48:49.384 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:49.376+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:48:50.363 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.356+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:48:50.363 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.356+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28c30 2026-02-19T20:48:50.363 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.356+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:48:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 565291008 bytes 2026-02-19T20:48:50.830 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.820+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4270 2026-02-19T20:48:50.831 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.824+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:48:50.831 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.824+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:48:50.831 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.824+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:48:50.831 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.824+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:48:50.833 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.824+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:48:50.833 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.824+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:48:50.833 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.824+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:48:50.833 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.824+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:48:50.834 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.824+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:48:50.834 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:50.824+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 586598400 bytes 2026-02-19T20:48:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 597284864 bytes 2026-02-19T20:48:51.482 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:51.472+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2f790 2026-02-19T20:48:51.482 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:51.476+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fca90 2026-02-19T20:48:51.483 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:51.476+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fdd40 2026-02-19T20:48:51.483 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:51.476+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396c30 2026-02-19T20:48:51.483 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:51.476+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:48:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:48:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_35 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 journal 128M 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:48:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_35 128M 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:48:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:48:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:48:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:48:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_35 2026-02-19T20:48:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:48:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:48:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:48:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:48:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:48:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_35 2026-02-19T20:48:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:48:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:48:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:48:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:48:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=9 2026-02-19T20:48:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:48:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:48:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 605184 bytes 2026-02-19T20:48:52.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 14096384 bytes 2026-02-19T20:48:52.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 20925440 bytes 2026-02-19T20:48:53.083 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.076+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:48:53.084 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.076+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:48:53.086 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.076+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:48:53.086 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.076+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:48:53.086 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.076+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:48:53.086 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.076+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:48:53.086 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.076+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 47143936 bytes 2026-02-19T20:48:53.400 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.392+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:48:53.400 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.392+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:48:53.400 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.392+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1da00 2026-02-19T20:48:53.400 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.392+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:48:53.401 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.392+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:48:53.401 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.392+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:53.401 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.392+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:48:53.401 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.392+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:53.401 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:53.392+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:53.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 68706304 bytes 2026-02-19T20:48:53.561 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:53.552+0000 7fd63ae12640 -1 reset not still connected to 0x55f10fae8b60 2026-02-19T20:48:53.562 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:53.552+0000 7fd63ae12640 -1 reset not still connected to 0x55f10fae9ba0 2026-02-19T20:48:53.562 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:53.552+0000 7fd63ae12640 -1 reset not still connected to 0x55f10fec65b0 2026-02-19T20:48:53.562 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:53.552+0000 7fd63ae12640 -1 reset not still connected to 0x55f11003bc70 2026-02-19T20:48:53.563 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:53.552+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74a90 2026-02-19T20:48:53.563 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:53.552+0000 7fd63ae12640 -1 reset not still connected to 0x55f123850a90 2026-02-19T20:48:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 69320704 bytes 2026-02-19T20:48:53.919 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:53.912+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:48:53.919 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:53.912+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:48:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 102467584 bytes 2026-02-19T20:48:54.699 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:54.692+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8c9c0 2026-02-19T20:48:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 140353536 bytes 2026-02-19T20:48:55.226 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:55.216+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:55.563 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:55.556+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:48:55.565 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:55.556+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:48:55.565 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:55.556+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1da00 2026-02-19T20:48:55.565 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:55.556+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:48:55.565 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:55.556+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:48:55.565 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:55.556+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:48:55.566 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:55.560+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:48:55.566 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:55.560+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:48:55.566 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:55.560+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:55.780 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:55.772+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74a90 2026-02-19T20:48:55.780 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:55.772+0000 7fd63ae12640 -1 reset not still connected to 0x55f123850a90 2026-02-19T20:48:55.781 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:48:55.772+0000 7fd63ae12640 -1 reset not still connected to 0x55f13ef47d40 2026-02-19T20:48:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 219196416 bytes 2026-02-19T20:48:56.023 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:56.016+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:48:56.024 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:56.016+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:48:56.362 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6271592313494689) > weight (0.25) 2026-02-19T20:48:56.362 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8615231681360304) > weight (0.25) 2026-02-19T20:48:56.362 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-19T20:48:56.363 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-19T20:48:56.363 INFO:tasks.rbd_mirror_thrash:waiting for 9.901305275937021 secs before reviving daemons 2026-02-19T20:48:56.364 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T20:48:56.351+0000 7fe2eb5a2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 65137) UID: 1000 2026-02-19T20:48:56.959 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:56.952+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:48:57.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 353241088 bytes 2026-02-19T20:48:57.800 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.792+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:48:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 371074048 bytes 2026-02-19T20:48:57.950 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.940+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:48:57.950 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.940+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:48:57.951 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.944+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:48:57.951 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.944+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:48:57.951 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.944+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:48:57.951 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.944+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:48:57.952 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.944+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:48:57.952 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.944+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:48:57.952 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.944+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:48:57.952 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:48:57.944+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:48:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 388002816 bytes 2026-02-19T20:48:58.245 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.236+0000 7f93a8cad640 -1 reset not still connected to 0x556e347328f0 2026-02-19T20:48:58.245 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.236+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:48:58.245 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.236+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:48:58.246 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.240+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:48:58.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.240+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:48:58.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.240+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:48:58.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.240+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:58.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.240+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:48:58.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.240+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:48:58.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.240+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:58.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:58.240+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:48:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 520059904 bytes 2026-02-19T20:48:59.714 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:59.704+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:48:59.715 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:59.708+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:48:59.715 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:59.708+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:48:59.715 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:59.708+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:48:59.717 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:59.708+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:48:59.717 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:59.708+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:48:59.718 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:59.708+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:48:59.718 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:59.708+0000 7f93a8cad640 -1 reset not still connected to 0x556e31d37380 2026-02-19T20:48:59.718 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:59.708+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:48:59.718 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:48:59.708+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:49:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 554723328 bytes 2026-02-19T20:49:00.694 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.684+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4270 2026-02-19T20:49:00.694 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.684+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:49:00.694 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.684+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:49:00.695 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.688+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:49:00.696 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.688+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:49:00.696 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.688+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:49:00.696 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.688+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:49:00.697 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.688+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:49:00.697 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.688+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:49:00.697 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.688+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:49:00.697 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:00.688+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:49:01.074 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.064+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:49:01.074 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.064+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28c30 2026-02-19T20:49:01.075 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.064+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:49:01.130 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.120+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:49:01.130 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.120+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:49:01.130 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.120+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1da00 2026-02-19T20:49:01.130 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.120+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:49:01.130 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.120+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:49:01.130 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.120+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:49:01.130 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.120+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:49:01.130 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.120+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:49:01.130 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:01.120+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:49:01.194 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:01.184+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fca90 2026-02-19T20:49:01.194 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:01.184+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fdd40 2026-02-19T20:49:01.194 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:01.184+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396c30 2026-02-19T20:49:01.195 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:01.188+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:49:01.200 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:01.192+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fca90 2026-02-19T20:49:01.200 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:01.192+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fdd40 2026-02-19T20:49:01.200 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:01.192+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396c30 2026-02-19T20:49:01.200 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:01.192+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:49:01.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:49:01.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:49:01.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:49:01.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:49:01.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:49:01.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_36 2026-02-19T20:49:01.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 journal 128M 2026-02-19T20:49:01.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:01.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:01.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:01.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_36 128M 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:49:01.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:49:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:49:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:49:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_36 2026-02-19T20:49:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:49:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:49:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:49:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:49:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:49:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_36 2026-02-19T20:49:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:49:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:49:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=12 2026-02-19T20:49:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:49:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:49:01.692 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:01.684+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:49:01.692 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:01.684+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:49:02.007 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:02.000+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:49:02.009 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:02.000+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:49:02.009 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:02.000+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1da00 2026-02-19T20:49:02.010 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:02.000+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:49:02.010 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:02.000+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:49:02.010 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:02.000+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:49:02.010 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:02.000+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:49:02.301 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:02.292+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:49:02.301 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:02.292+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:49:02.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 73238528 bytes 2026-02-19T20:49:02.605 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:02.596+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:49:02.639 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:02.632+0000 7fd63ae12640 -1 reset not still connected to 0x55f1124ccdd0 2026-02-19T20:49:02.640 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:02.632+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:49:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 94479360 bytes 2026-02-19T20:49:03.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 130199552 bytes 2026-02-19T20:49:03.901 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:03.892+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74a90 2026-02-19T20:49:03.901 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:03.892+0000 7fd63ae12640 -1 reset not still connected to 0x55f123850a90 2026-02-19T20:49:04.321 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.312+0000 7f93a8cad640 -1 reset not still connected to 0x556e493de340 2026-02-19T20:49:04.323 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.316+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:49:04.323 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.316+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:49:04.351 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.344+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44750 2026-02-19T20:49:04.351 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.344+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44dd0 2026-02-19T20:49:04.351 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.344+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719450 2026-02-19T20:49:04.351 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.344+0000 7f93a8cad640 -1 reset not still connected to 0x556e34a28dd0 2026-02-19T20:49:04.352 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.344+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2f790 2026-02-19T20:49:04.352 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.344+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d996c0 2026-02-19T20:49:04.352 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.344+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fca90 2026-02-19T20:49:04.352 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.344+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fdd40 2026-02-19T20:49:04.353 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.344+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396c30 2026-02-19T20:49:04.353 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.344+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:49:04.388 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.380+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4270 2026-02-19T20:49:04.389 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.380+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:49:04.389 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.380+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:49:04.390 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.380+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:49:04.390 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.380+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:49:04.390 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.380+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:49:04.390 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.380+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:49:04.390 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.380+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:49:04.391 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.380+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:49:04.391 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.384+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:49:04.391 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:04.384+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:49:04.614 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.608+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d3301a0 2026-02-19T20:49:04.615 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.608+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d99110 2026-02-19T20:49:04.616 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.608+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d995f0 2026-02-19T20:49:04.616 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.608+0000 7f93a8cad640 -1 reset not still connected to 0x556e493de340 2026-02-19T20:49:04.616 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.608+0000 7f93a8cad640 -1 reset not still connected to 0x556e493dfad0 2026-02-19T20:49:04.617 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.608+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:49:04.617 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.608+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:49:04.953 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.944+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44820 2026-02-19T20:49:04.954 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.948+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:49:04.954 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:04.948+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:49:05.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 234387456 bytes 2026-02-19T20:49:05.399 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:05.392+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:49:05.399 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:05.392+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:49:05.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 246842368 bytes 2026-02-19T20:49:06.210 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:06.200+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:49:06.210 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:06.200+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:49:06.265 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-19T20:49:06.265 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:49:06.266 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-19T20:49:06.266 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-19T20:49:06.266 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-19T20:49:06.268 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-19T20:49:06.268 INFO:tasks.rbd_mirror_thrash:waiting for 71.07922811782657 secs before thrashing 2026-02-19T20:49:06.362 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:06.352+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fdd40 2026-02-19T20:49:06.362 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:06.352+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:49:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 307553280 bytes 2026-02-19T20:49:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 333521920 bytes 2026-02-19T20:49:07.390 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:07.380+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:49:07.392 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:07.384+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:49:07.393 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:07.384+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:49:07.393 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:07.384+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:49:07.393 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:07.384+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:49:07.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 386286592 bytes 2026-02-19T20:49:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 398253056 bytes 2026-02-19T20:49:08.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 420608000 bytes 2026-02-19T20:49:08.698 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:08.688+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:49:08.699 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:08.692+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:49:08.700 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:08.692+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:49:08.700 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:08.692+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:49:08.700 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:08.692+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:49:09.061 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:09.052+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:49:09.062 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:09.052+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:49:09.201 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.192+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:49:09.202 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.192+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:49:09.202 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.192+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:49:09.202 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.192+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:49:09.203 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.192+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:49:09.203 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.192+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:49:09.203 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.192+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:49:09.203 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.192+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:49:09.228 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.220+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:49:09.302 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.292+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:49:09.302 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.296+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:49:09.304 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.296+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:49:09.304 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.296+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:49:09.304 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.296+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:49:09.305 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.296+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:49:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 469649408 bytes 2026-02-19T20:49:09.456 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.448+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:49:09.457 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.448+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:49:09.457 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.448+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:49:09.458 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.448+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:49:09.459 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.452+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:49:09.459 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.452+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:49:09.460 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.452+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:49:09.460 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.452+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:49:09.460 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.452+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:49:09.460 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:09.452+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:49:09.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 482297856 bytes 2026-02-19T20:49:10.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 511599616 bytes 2026-02-19T20:49:10.331 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:10.324+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2f790 2026-02-19T20:49:10.332 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:10.324+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d996c0 2026-02-19T20:49:10.332 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:10.324+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fca90 2026-02-19T20:49:10.332 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:10.324+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fdd40 2026-02-19T20:49:10.333 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:10.324+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396c30 2026-02-19T20:49:10.333 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:10.324+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:49:10.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 555091968 bytes 2026-02-19T20:49:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 581995520 bytes 2026-02-19T20:49:11.573 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:11.564+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:49:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 602455040 bytes 2026-02-19T20:49:11.861 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:11.852+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:49:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 662013952 bytes 2026-02-19T20:49:12.644 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:12.636+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:49:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 706905088 bytes 2026-02-19T20:49:13.103 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:13.096+0000 7fd63ae12640 -1 reset not still connected to 0x55f114c74a90 2026-02-19T20:49:13.104 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:13.096+0000 7fd63ae12640 -1 reset not still connected to 0x55f123850a90 2026-02-19T20:49:13.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 710966272 bytes 2026-02-19T20:49:13.288 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.280+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:49:13.289 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.280+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:49:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:49:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:49:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:49:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:49:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_37 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 journal 128M 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:49:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_37 128M 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:13.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_37 2026-02-19T20:49:13.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:49:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:49:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:49:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:49:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:49:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_37 2026-02-19T20:49:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T20:49:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:49:13.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=12 2026-02-19T20:49:13.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:49:13.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:49:13.675 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.664+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb2a90 2026-02-19T20:49:13.675 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.664+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb64410 2026-02-19T20:49:13.675 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.664+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb65930 2026-02-19T20:49:13.675 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.664+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d3301a0 2026-02-19T20:49:13.676 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.664+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d99110 2026-02-19T20:49:13.676 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.664+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d995f0 2026-02-19T20:49:13.676 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.664+0000 7f93a8cad640 -1 reset not still connected to 0x556e493de340 2026-02-19T20:49:13.676 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.664+0000 7f93a8cad640 -1 reset not still connected to 0x556e493dfad0 2026-02-19T20:49:13.676 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.664+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:49:13.676 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:13.668+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:49:13.800 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:13.792+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:49:13.800 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:13.792+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:49:13.801 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:13.792+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:49:14.092 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:14.084+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:49:14.092 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:14.084+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:49:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 51348480 bytes 2026-02-19T20:49:14.770 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:14.760+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:49:14.771 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:14.764+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:49:14.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 107491328 bytes 2026-02-19T20:49:15.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 143557632 bytes 2026-02-19T20:49:15.539 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.528+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:49:15.539 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.532+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:49:15.539 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.532+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:49:15.539 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.532+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:49:15.539 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.532+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:49:15.540 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.532+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:49:15.737 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:15.728+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:49:15.738 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:15.728+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:49:15.738 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:15.728+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:49:15.738 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:15.728+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:49:15.738 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:15.728+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:49:15.739 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:15.728+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:49:15.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 170546176 bytes 2026-02-19T20:49:15.970 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.960+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00d08340 2026-02-19T20:49:15.971 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.964+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0b3104e0 2026-02-19T20:49:15.971 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.964+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0d00 2026-02-19T20:49:15.972 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.964+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:49:15.972 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.964+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:49:15.972 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.964+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28c30 2026-02-19T20:49:15.972 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:15.964+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:49:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 183615488 bytes 2026-02-19T20:49:16.763 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:49:16.751+0000 7f5ce3652640 -1 reset not still connected to 0x56104e4e48f0 2026-02-19T20:49:16.763 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:49:16.751+0000 7f5ce3652640 -1 reset not still connected to 0x5610527ec820 2026-02-19T20:49:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 250955776 bytes 2026-02-19T20:49:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 297114624 bytes 2026-02-19T20:49:17.901 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:17.892+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:49:17.901 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:17.892+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28c30 2026-02-19T20:49:17.901 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:17.892+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:49:18.315 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.304+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb65930 2026-02-19T20:49:18.316 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.304+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d3301a0 2026-02-19T20:49:18.316 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.304+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d99110 2026-02-19T20:49:18.316 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.304+0000 7f93a8cad640 -1 reset not still connected to 0x556e493de340 2026-02-19T20:49:18.316 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.304+0000 7f93a8cad640 -1 reset not still connected to 0x556e493dfad0 2026-02-19T20:49:18.316 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.304+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:49:18.317 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.304+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:49:18.582 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.572+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:49:18.582 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.572+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:49:18.583 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.576+0000 7f93a8cad640 -1 reset not still connected to 0x556e31d37380 2026-02-19T20:49:18.583 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.576+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44820 2026-02-19T20:49:18.583 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.576+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:49:18.583 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:18.576+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:49:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 363207680 bytes 2026-02-19T20:49:18.688 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:18.680+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:49:19.118 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x5649fdd13790 2026-02-19T20:49:19.118 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4270 2026-02-19T20:49:19.119 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:49:19.119 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:49:19.119 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:49:19.119 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:49:19.119 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:49:19.119 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:49:19.119 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:49:19.119 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:49:19.119 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:49:19.119 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.108+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:49:19.168 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.160+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:49:19.170 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:19.160+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:49:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 404125696 bytes 2026-02-19T20:49:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 443295744 bytes 2026-02-19T20:49:20.100 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:20.092+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:49:20.101 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:20.092+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50ea0 2026-02-19T20:49:20.101 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:20.092+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:49:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 463906816 bytes 2026-02-19T20:49:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 572949504 bytes 2026-02-19T20:49:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 605147136 bytes 2026-02-19T20:49:23.408 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:23.400+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0d00 2026-02-19T20:49:23.408 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:23.400+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:49:23.408 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:23.400+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:49:23.409 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:23.400+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28c30 2026-02-19T20:49:23.409 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:23.400+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:49:23.637 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:23.628+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a2820 2026-02-19T20:49:24.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 737512448 bytes 2026-02-19T20:49:24.589 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:24.580+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00d08340 2026-02-19T20:49:24.591 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:24.584+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0b3104e0 2026-02-19T20:49:24.591 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:24.584+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb0d00 2026-02-19T20:49:24.591 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:24.584+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3a410 2026-02-19T20:49:24.591 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:24.584+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11d3b040 2026-02-19T20:49:24.591 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:24.584+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28c30 2026-02-19T20:49:24.591 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:24.584+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f23860 2026-02-19T20:49:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 764255232 bytes 2026-02-19T20:49:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:49:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:49:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:49:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:49:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:49:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_38 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 journal 128M 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_38 128M 2026-02-19T20:49:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T20:49:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:49:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:49:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:49:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_38 2026-02-19T20:49:25.468 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T20:49:25.455+0000 7f5ce3652640 -1 reset not still connected to 0x561036b09110 2026-02-19T20:49:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:49:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:49:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:49:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:49:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:49:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_38 2026-02-19T20:49:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T20:49:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:49:25.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=7 2026-02-19T20:49:25.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:49:25.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:49:25.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 11462656 bytes 2026-02-19T20:49:26.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 56157184 bytes 2026-02-19T20:49:26.466 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:26.456+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fca90 2026-02-19T20:49:26.466 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:26.456+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fdd40 2026-02-19T20:49:26.466 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:26.456+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396c30 2026-02-19T20:49:26.466 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:26.456+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:49:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 82777088 bytes 2026-02-19T20:49:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 141351936 bytes 2026-02-19T20:49:29.416 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:29.408+0000 7f93a8cad640 -1 reset not still connected to 0x556e452fdd40 2026-02-19T20:49:29.417 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:29.408+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396c30 2026-02-19T20:49:29.417 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:29.408+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e31e0 2026-02-19T20:49:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 202110976 bytes 2026-02-19T20:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 210227200 bytes 2026-02-19T20:49:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 210833408 bytes 2026-02-19T20:49:31.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 234045440 bytes 2026-02-19T20:49:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 258774016 bytes 2026-02-19T20:49:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:49:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:49:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:49:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:49:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:49:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_39 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 journal 128M 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_39 128M 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_39 2026-02-19T20:49:32.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:49:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:49:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:49:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:49:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:49:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_39 2026-02-19T20:49:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T20:49:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:49:32.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=11 2026-02-19T20:49:32.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:49:32.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:49:33.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 66404352 bytes 2026-02-19T20:49:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 104288256 bytes 2026-02-19T20:49:34.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 127172608 bytes 2026-02-19T20:49:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 144114688 bytes 2026-02-19T20:49:35.202 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4095028019617155) > weight (0.25) 2026-02-19T20:49:35.203 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.44207288932391575) > weight (0.25) 2026-02-19T20:49:35.203 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-19T20:49:35.203 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-19T20:49:35.203 INFO:tasks.rbd_mirror_thrash:waiting for 2.6631626298526765 secs before reviving daemons 2026-02-19T20:49:35.204 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T20:49:35.191+0000 7ff4b8503640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 75646) UID: 1000 2026-02-19T20:49:36.035 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:36.028+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:49:36.038 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:36.028+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:49:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 239143936 bytes 2026-02-19T20:49:36.211 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:36.204+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:49:36.211 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:36.204+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:49:36.212 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:36.204+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:49:36.906 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:36.896+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:49:36.906 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:36.896+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a2680 2026-02-19T20:49:36.930 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:36.920+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:49:36.933 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:36.924+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:49:37.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 327299072 bytes 2026-02-19T20:49:37.867 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-19T20:49:37.867 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:49:37.867 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-19T20:49:37.867 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-19T20:49:37.867 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-19T20:49:37.869 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-19T20:49:37.869 INFO:tasks.rbd_mirror_thrash:waiting for 102.00655838821524 secs before thrashing 2026-02-19T20:49:38.526 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:38.516+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:49:38.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 413052928 bytes 2026-02-19T20:49:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 498440192 bytes 2026-02-19T20:49:39.986 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:39.976+0000 7fe4dd68c640 -1 reset not still connected to 0x564a009ad1e0 2026-02-19T20:49:39.986 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:39.976+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:49:39.986 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:39.976+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:49:39.986 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:39.976+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:49:39.986 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:39.976+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:49:39.987 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:39.976+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:49:40.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 527626240 bytes 2026-02-19T20:49:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 533787648 bytes 2026-02-19T20:49:40.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 562200576 bytes 2026-02-19T20:49:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 604006400 bytes 2026-02-19T20:49:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 687034368 bytes 2026-02-19T20:49:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 717464576 bytes 2026-02-19T20:49:43.155 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:43.148+0000 7fd63ae12640 -1 reset not still connected to 0x55f112563860 2026-02-19T20:49:43.156 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:43.148+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:49:43.156 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:49:43.148+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a2680 2026-02-19T20:49:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 738157568 bytes 2026-02-19T20:49:43.434 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:49:43.423+0000 7efcbd1bd640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x560a7b1cae00 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-19T20:49:43.435 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:49:43.423+0000 7efcbd1bd640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x560a7b1cae00 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-19T20:49:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 747759616 bytes 2026-02-19T20:49:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:49:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:49:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:49:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_40 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 journal 128M 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_40 128M 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:43.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T20:49:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:49:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:49:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:49:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:43.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_40 2026-02-19T20:49:43.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:49:43.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:43.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:49:43.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:49:43.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:49:43.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:49:43.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_40 2026-02-19T20:49:43.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:43.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:43.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T20:49:43.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:49:43.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=12 2026-02-19T20:49:43.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:49:43.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:49:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 34128896 bytes 2026-02-19T20:49:44.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 56220672 bytes 2026-02-19T20:49:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 66627584 bytes 2026-02-19T20:49:45.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 104221696 bytes 2026-02-19T20:49:46.227 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:46.224+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:49:46.227 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:46.224+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:49:46.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:46.224+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:49:47.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 147317760 bytes 2026-02-19T20:49:47.710 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:47.704+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:49:47.711 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:47.708+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:49:47.712 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:47.708+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:49:47.713 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:47.708+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:49:47.713 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:47.708+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:49:47.713 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:47.708+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:49:48.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 201986048 bytes 2026-02-19T20:49:48.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 219371520 bytes 2026-02-19T20:49:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 256847872 bytes 2026-02-19T20:49:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 261462016 bytes 2026-02-19T20:49:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 306440192 bytes 2026-02-19T20:49:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 340988928 bytes 2026-02-19T20:49:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 380891136 bytes 2026-02-19T20:49:51.387 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:51.383+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:49:51.387 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:51.383+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:49:51.387 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:51.383+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:49:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 403578880 bytes 2026-02-19T20:49:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 430256128 bytes 2026-02-19T20:49:52.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 500188160 bytes 2026-02-19T20:49:53.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 527713280 bytes 2026-02-19T20:49:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 528321536 bytes 2026-02-19T20:49:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 549904384 bytes 2026-02-19T20:49:54.147 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:54.143+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44820 2026-02-19T20:49:54.147 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:54.143+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45790 2026-02-19T20:49:54.148 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:54.143+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:49:54.148 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:54.143+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:49:54.148 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:54.143+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:49:54.148 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:54.143+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:49:54.148 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:54.143+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:49:54.297 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:54.291+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:49:54.297 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:54.291+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:49:54.400 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:54.395+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:49:54.401 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:54.395+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:49:54.401 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:54.395+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:49:54.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 607799296 bytes 2026-02-19T20:49:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2000 IOs, 640587776 bytes 2026-02-19T20:49:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:49:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:49:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:49:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:49:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:49:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_41 2026-02-19T20:49:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 journal 128M 2026-02-19T20:49:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_41 128M 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:55.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T20:49:55.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:55.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:49:55.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:49:55.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:49:55.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:49:55.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_41 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_41 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T20:49:55.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:49:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=6 2026-02-19T20:49:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:49:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:49:55.976 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:55.971+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44820 2026-02-19T20:49:55.976 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:55.971+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45790 2026-02-19T20:49:55.977 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:55.971+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:49:55.978 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:55.971+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:49:55.978 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:55.971+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:49:55.978 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:55.971+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:49:55.978 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:49:55.971+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:49:56.520 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:56.515+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:49:56.522 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:49:56.515+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:49:56.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 30354432 bytes 2026-02-19T20:49:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 101638144 bytes 2026-02-19T20:49:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 108990464 bytes 2026-02-19T20:49:59.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 149975040 bytes 2026-02-19T20:49:59.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 172537856 bytes 2026-02-19T20:50:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 189357056 bytes 2026-02-19T20:50:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 222569472 bytes 2026-02-19T20:50:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 248922112 bytes 2026-02-19T20:50:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:50:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:50:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:50:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:50:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:50:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_42 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 journal 128M 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_42 128M 2026-02-19T20:50:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T20:50:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:50:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:50:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:50:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_42 2026-02-19T20:50:01.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:50:01.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:50:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:50:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:50:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:50:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_42 2026-02-19T20:50:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T20:50:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:50:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=12 2026-02-19T20:50:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:50:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:50:01.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 2988032 bytes 2026-02-19T20:50:02.203 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:02.199+0000 7fd63ae12640 -1 reset not still connected to 0x55f112563860 2026-02-19T20:50:02.203 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:02.199+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:50:02.203 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:02.199+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a2680 2026-02-19T20:50:02.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 86764544 bytes 2026-02-19T20:50:03.422 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:03.415+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:03.423 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:03.415+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:03.423 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:03.419+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:03.549 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:03.543+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:50:03.550 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:03.543+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:03.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 123713536 bytes 2026-02-19T20:50:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 183629824 bytes 2026-02-19T20:50:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 209814528 bytes 2026-02-19T20:50:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 270959616 bytes 2026-02-19T20:50:06.496 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:06.491+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:50:06.496 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:06.491+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:50:06.498 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:06.491+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:06.498 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:06.491+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:06.498 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:06.491+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:06.498 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:06.491+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 310561792 bytes 2026-02-19T20:50:07.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 383448064 bytes 2026-02-19T20:50:08.058 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:08.051+0000 7fe4dd68c640 -1 reset not still connected to 0x564a009ad1e0 2026-02-19T20:50:08.058 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:08.051+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97930 2026-02-19T20:50:08.058 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:08.051+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:08.058 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:08.051+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:08.058 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:08.051+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:08.058 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:08.051+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 398546944 bytes 2026-02-19T20:50:09.104 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.099+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:50:09.105 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.099+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:50:09.105 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.099+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:50:09.106 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.099+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:09.106 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.099+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:09.106 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.099+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:09.106 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.099+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 485844992 bytes 2026-02-19T20:50:09.387 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:09.383+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:09.388 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:09.383+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:09.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 495610880 bytes 2026-02-19T20:50:09.951 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.947+0000 7f93a8cad640 -1 reset not still connected to 0x556e31eba270 2026-02-19T20:50:09.951 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.947+0000 7f93a8cad640 -1 reset not still connected to 0x556e31ebb6c0 2026-02-19T20:50:09.952 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.947+0000 7f93a8cad640 -1 reset not still connected to 0x556e32496a90 2026-02-19T20:50:09.952 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.947+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275cf70 2026-02-19T20:50:09.952 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.947+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:50:09.952 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.947+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:50:09.952 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.947+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:09.952 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.947+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:09.952 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:09.947+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:10.479 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:10.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:10.482 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:10.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:10.482 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:10.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:50:10.482 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:10.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:10.483 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:10.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:10.483 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:10.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:10.483 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:10.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:10.519 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:10.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e31eba270 2026-02-19T20:50:10.519 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:10.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e31ebb6c0 2026-02-19T20:50:10.520 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:10.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e32496a90 2026-02-19T20:50:10.520 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:10.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275cf70 2026-02-19T20:50:10.520 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:10.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:50:10.521 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:10.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:50:10.521 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:10.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:10.521 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:10.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:10.521 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:10.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:10.531 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:10.527+0000 7fd63ae12640 -1 reset not still connected to 0x55f1124cc4e0 2026-02-19T20:50:10.531 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:10.527+0000 7fd63ae12640 -1 reset not still connected to 0x55f1124ccdd0 2026-02-19T20:50:10.532 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:10.527+0000 7fd63ae12640 -1 reset not still connected to 0x55f112563860 2026-02-19T20:50:10.532 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:10.527+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:50:10.532 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:10.527+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a2680 2026-02-19T20:50:10.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 592326656 bytes 2026-02-19T20:50:11.130 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:11.123+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:50:11.131 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:11.127+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:11.131 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:11.127+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:11.131 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:11.127+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:11.131 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:11.127+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:11.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 610403328 bytes 2026-02-19T20:50:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 645069824 bytes 2026-02-19T20:50:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 695998464 bytes 2026-02-19T20:50:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 710621184 bytes 2026-02-19T20:50:13.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:50:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:50:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:50:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:50:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:50:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_43 2026-02-19T20:50:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 journal 128M 2026-02-19T20:50:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_43 128M 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T20:50:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:50:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:50:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:50:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_43 2026-02-19T20:50:13.734 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:13.727+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:50:13.735 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:13.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:50:13.735 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:13.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:50:13.735 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:13.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:13.735 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:13.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:13.736 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:13.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:13.737 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:13.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:50:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:50:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:50:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:50:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:50:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_43 2026-02-19T20:50:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T20:50:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:50:13.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=13 2026-02-19T20:50:13.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:50:13.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:50:13.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 601088 bytes 2026-02-19T20:50:14.944 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:14.939+0000 7fe4dd68c640 -1 reset not still connected to 0x564a009ad1e0 2026-02-19T20:50:14.945 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:14.939+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97930 2026-02-19T20:50:14.945 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:14.939+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:14.945 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:14.939+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:14.946 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:14.939+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:50:14.946 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:14.939+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:14.946 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:14.939+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:14.946 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:14.939+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:14.946 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:14.939+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 51099648 bytes 2026-02-19T20:50:16.805 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:16.799+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:16.805 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:16.799+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:16.807 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:16.803+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:50:16.807 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:16.803+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:16.807 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:16.803+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:16.807 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:16.803+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:16.808 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:16.803+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:17.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 104003584 bytes 2026-02-19T20:50:17.289 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:17.283+0000 7fd63ae12640 -1 reset not still connected to 0x55f111349860 2026-02-19T20:50:17.290 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:17.283+0000 7fd63ae12640 -1 reset not still connected to 0x55f111fd1ba0 2026-02-19T20:50:17.290 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:17.283+0000 7fd63ae12640 -1 reset not still connected to 0x55f1125632b0 2026-02-19T20:50:17.290 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:17.283+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d944e0 2026-02-19T20:50:17.290 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:17.283+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d95ee0 2026-02-19T20:50:17.292 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:17.287+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b24f70 2026-02-19T20:50:17.292 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:17.287+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b25e10 2026-02-19T20:50:17.348 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.26566022177083093) > weight (0.25) 2026-02-19T20:50:17.348 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6016764037043236) > weight (0.25) 2026-02-19T20:50:17.348 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6300467348332239) > weight (0.25) 2026-02-19T20:50:17.349 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-19T20:50:17.349 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-19T20:50:17.349 INFO:tasks.rbd_mirror_thrash:waiting for 1.8950083382018923 secs before reviving daemons 2026-02-19T20:50:17.349 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T20:50:17.339+0000 7f6696a0f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 77984) UID: 1000 2026-02-19T20:50:17.731 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:17.727+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45790 2026-02-19T20:50:17.731 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:17.727+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2f520 2026-02-19T20:50:17.732 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:17.727+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:50:17.732 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:17.727+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:50:17.732 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:17.727+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:50:17.734 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:17.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:17.734 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:17.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:17.735 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:17.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:50:17.735 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:17.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:17.735 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:17.731+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 152916992 bytes 2026-02-19T20:50:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 163488768 bytes 2026-02-19T20:50:18.854 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:18.847+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:18.854 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:18.847+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 200896512 bytes 2026-02-19T20:50:19.245 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-19T20:50:19.245 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:50:19.246 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-19T20:50:19.246 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-19T20:50:19.246 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-19T20:50:19.248 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-19T20:50:19.248 INFO:tasks.rbd_mirror_thrash:waiting for 95.23778411399536 secs before thrashing 2026-02-19T20:50:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 216126464 bytes 2026-02-19T20:50:19.338 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:19.331+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b24f70 2026-02-19T20:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 242170880 bytes 2026-02-19T20:50:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 277910528 bytes 2026-02-19T20:50:21.878 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:21.871+0000 7fd63ae12640 -1 reset not still connected to 0x55f112563860 2026-02-19T20:50:21.878 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:21.871+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:50:21.878 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:21.871+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a2680 2026-02-19T20:50:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 356441088 bytes 2026-02-19T20:50:22.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 360332288 bytes 2026-02-19T20:50:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 362401792 bytes 2026-02-19T20:50:22.286 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:22.283+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:22.668 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:22.663+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 411076608 bytes 2026-02-19T20:50:24.025 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:24.019+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:24.026 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:24.019+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:24.027 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:24.023+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:24.027 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:24.023+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:24.027 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:24.023+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:24.029 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:24.023+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:24.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 484331520 bytes 2026-02-19T20:50:24.433 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:24.427+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:50:24.435 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:24.427+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:50:24.435 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:24.427+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:50:24.435 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:24.427+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:24.435 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:24.427+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:24.435 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:24.431+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:50:24.436 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:24.431+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:24.436 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:24.431+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:25.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 527834112 bytes 2026-02-19T20:50:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 556013568 bytes 2026-02-19T20:50:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:50:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:50:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:50:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:50:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:50:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_44 2026-02-19T20:50:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 journal 128M 2026-02-19T20:50:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_44 128M 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T20:50:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:50:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:50:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:50:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_44 2026-02-19T20:50:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:50:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:50:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:50:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:50:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:50:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_44 2026-02-19T20:50:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T20:50:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:50:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=8 2026-02-19T20:50:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:50:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:50:27.078 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:27.071+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275cf70 2026-02-19T20:50:27.078 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:27.071+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:50:27.079 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:27.071+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:50:27.079 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:27.071+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:27.079 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:27.071+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:27.080 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:27.075+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:27.187 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:27.183+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:50:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 76316672 bytes 2026-02-19T20:50:28.482 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:28.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:28.482 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:28.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:28.482 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:28.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:28.482 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:28.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:28.483 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:28.479+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 118596608 bytes 2026-02-19T20:50:29.214 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:29.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:29.214 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:29.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:29.214 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:29.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:29.215 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:29.211+0000 7f93a8cad640 -1 reset not still connected to 0x556e747ab6c0 2026-02-19T20:50:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 121932800 bytes 2026-02-19T20:50:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 146033664 bytes 2026-02-19T20:50:31.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 202808320 bytes 2026-02-19T20:50:31.619 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:31.615+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:31.619 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:31.615+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:31.619 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:31.615+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:50:31.620 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:31.615+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:31.620 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:31.615+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:32.544 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:32.539+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:32.545 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:32.539+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:32.548 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:32.543+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:50:32.548 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:32.543+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:32.548 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:32.543+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:32.548 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:32.543+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:32.548 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:32.543+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 280884224 bytes 2026-02-19T20:50:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 342949888 bytes 2026-02-19T20:50:34.145 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:34.139+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275cf70 2026-02-19T20:50:34.146 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:34.139+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:50:34.146 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:34.139+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:50:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:50:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:50:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:50:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:50:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_45 2026-02-19T20:50:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 journal 128M 2026-02-19T20:50:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_45 128M 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:50:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:50:34.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:50:34.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:34.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_45 2026-02-19T20:50:34.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:50:34.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:34.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:50:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:50:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:50:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:50:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_45 2026-02-19T20:50:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T20:50:34.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:50:34.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=14 2026-02-19T20:50:34.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:50:34.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:50:34.958 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:34.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:34.959 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:34.955+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:35.818 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:35.811+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:35.819 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:35.815+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:50:35.819 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:35.815+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:35.820 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:35.815+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:35.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 69374976 bytes 2026-02-19T20:50:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 74498048 bytes 2026-02-19T20:50:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 91855872 bytes 2026-02-19T20:50:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 107351040 bytes 2026-02-19T20:50:36.874 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.867+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd5a00 2026-02-19T20:50:36.874 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.867+0000 7fe4dd68c640 -1 reset not still connected to 0x564a009ad1e0 2026-02-19T20:50:36.876 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.871+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97930 2026-02-19T20:50:36.876 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.871+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:36.876 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.871+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd9860 2026-02-19T20:50:36.877 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.871+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:36.877 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.871+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:50:36.877 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.871+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:36.877 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.871+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:36.877 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.871+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:36.877 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.871+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:36.995 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.991+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:36.996 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.991+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:36.997 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.991+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:50:36.997 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.991+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:36.997 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.991+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:36.997 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.991+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:36.998 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:36.991+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:37.454 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:37.447+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:37.481 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:37.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:37.481 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:37.475+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:37.483 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:37.479+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:37.484 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:37.479+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:37.484 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:37.479+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:37.484 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:37.479+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:37.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 143332352 bytes 2026-02-19T20:50:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 167956480 bytes 2026-02-19T20:50:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 182721536 bytes 2026-02-19T20:50:39.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 209530880 bytes 2026-02-19T20:50:39.503 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:39.499+0000 7fd63ae12640 -1 reset not still connected to 0x55f111349860 2026-02-19T20:50:39.503 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:39.499+0000 7fd63ae12640 -1 reset not still connected to 0x55f111fd1ba0 2026-02-19T20:50:39.503 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:39.499+0000 7fd63ae12640 -1 reset not still connected to 0x55f1125632b0 2026-02-19T20:50:39.505 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:39.499+0000 7fd63ae12640 -1 reset not still connected to 0x55f1174ca9c0 2026-02-19T20:50:39.505 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:39.499+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d944e0 2026-02-19T20:50:39.505 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:39.499+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d95ee0 2026-02-19T20:50:39.505 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:39.499+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b24f70 2026-02-19T20:50:39.505 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:39.499+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b25e10 2026-02-19T20:50:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 234374144 bytes 2026-02-19T20:50:39.958 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:39.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45790 2026-02-19T20:50:39.958 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:39.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2f520 2026-02-19T20:50:39.959 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:39.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:50:39.959 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:39.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:50:39.959 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:39.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:50:39.960 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:39.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:39.960 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:39.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:39.960 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:39.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:50:39.960 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:39.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:39.960 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:39.951+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:40.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 310986752 bytes 2026-02-19T20:50:40.788 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:40.783+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:40.788 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:40.783+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:41.178 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.171+0000 7f93a8cad640 -1 reset not still connected to 0x556e31eba270 2026-02-19T20:50:41.178 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.171+0000 7f93a8cad640 -1 reset not still connected to 0x556e31ebb6c0 2026-02-19T20:50:41.179 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.171+0000 7f93a8cad640 -1 reset not still connected to 0x556e32496a90 2026-02-19T20:50:41.179 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.171+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275cf70 2026-02-19T20:50:41.179 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.171+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:50:41.179 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.171+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:50:41.179 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.171+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:41.179 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.171+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:41.179 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.171+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:41.179 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.171+0000 7f93a8cad640 -1 reset not still connected to 0x556e747ab6c0 2026-02-19T20:50:41.212 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e31eba270 2026-02-19T20:50:41.212 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e31ebb6c0 2026-02-19T20:50:41.213 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e32496a90 2026-02-19T20:50:41.213 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275cf70 2026-02-19T20:50:41.213 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:50:41.213 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:50:41.213 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:41.214 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:41.214 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:41.214 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.207+0000 7f93a8cad640 -1 reset not still connected to 0x556e747ab6c0 2026-02-19T20:50:41.245 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.239+0000 7f93a8cad640 -1 reset not still connected to 0x556e31eba270 2026-02-19T20:50:41.245 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.239+0000 7f93a8cad640 -1 reset not still connected to 0x556e31ebb6c0 2026-02-19T20:50:41.246 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.239+0000 7f93a8cad640 -1 reset not still connected to 0x556e32496a90 2026-02-19T20:50:41.246 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.239+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275cf70 2026-02-19T20:50:41.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.239+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:50:41.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.239+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:50:41.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.239+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:41.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.239+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:41.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.239+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:41.247 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:41.239+0000 7f93a8cad640 -1 reset not still connected to 0x556e747ab6c0 2026-02-19T20:50:41.331 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:41.327+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24147ad0 2026-02-19T20:50:41.331 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:41.327+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6820 2026-02-19T20:50:41.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 409560064 bytes 2026-02-19T20:50:42.615 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:42.607+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:50:42.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 434529280 bytes 2026-02-19T20:50:42.846 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:42.839+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:42.846 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:42.843+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:42.848 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:42.843+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:50:42.848 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:42.843+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:42.848 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:42.843+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:42.848 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:42.843+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:42.850 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:42.843+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:43.410 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:43.403+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:43.411 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:43.403+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:43.411 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:43.403+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:50:43.411 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:43.403+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:43.411 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:43.407+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:43.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 469549056 bytes 2026-02-19T20:50:43.948 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:43.943+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:50:43.948 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:43.943+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:50:43.948 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:43.943+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:50:43.949 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:43.943+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:50:43.950 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:43.943+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:50:43.950 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:43.943+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:50:43.950 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:43.943+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:50:43.950 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:43.943+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:50:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 488242176 bytes 2026-02-19T20:50:44.593 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:44.587+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:44.626 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:44.619+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:44.626 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:44.623+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:44.626 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:44.623+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:44.628 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:44.623+0000 7f93a8cad640 -1 reset not still connected to 0x556e747ab6c0 2026-02-19T20:50:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 552744960 bytes 2026-02-19T20:50:46.209 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:46.203+0000 7fd63ae12640 -1 reset not still connected to 0x55f110e175f0 2026-02-19T20:50:46.209 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:46.203+0000 7fd63ae12640 -1 reset not still connected to 0x55f116bb31e0 2026-02-19T20:50:46.209 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:46.203+0000 7fd63ae12640 -1 reset not still connected to 0x55f13ca611e0 2026-02-19T20:50:46.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 593212416 bytes 2026-02-19T20:50:46.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 615127040 bytes 2026-02-19T20:50:46.469 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:46.463+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50680 2026-02-19T20:50:46.471 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:46.467+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb01a0 2026-02-19T20:50:46.471 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:46.467+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738dd0 2026-02-19T20:50:46.471 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:46.467+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24147ad0 2026-02-19T20:50:46.471 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:46.467+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6820 2026-02-19T20:50:46.492 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:46.487+0000 7fd63ae12640 -1 reset not still connected to 0x55f110e175f0 2026-02-19T20:50:46.493 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:46.487+0000 7fd63ae12640 -1 reset not still connected to 0x55f116bb31e0 2026-02-19T20:50:46.493 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:46.487+0000 7fd63ae12640 -1 reset not still connected to 0x55f13ca611e0 2026-02-19T20:50:47.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 677171200 bytes 2026-02-19T20:50:47.524 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:47.519+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:50:47.525 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:47.519+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:50:47.525 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:47.519+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:50:47.525 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:47.519+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:47.525 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:47.519+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:47.526 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:47.519+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:50:47.526 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:47.519+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:47.526 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:47.519+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 743058432 bytes 2026-02-19T20:50:48.435 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.431+0000 7fe4dd68c640 -1 reset not still connected to 0x564a009ad1e0 2026-02-19T20:50:48.437 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.431+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97930 2026-02-19T20:50:48.437 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.431+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:48.437 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.431+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:48.438 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.431+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:50:48.439 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.431+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:48.439 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.431+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:48.439 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.431+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:48.439 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.435+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:48.472 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.467+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:50:48.476 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.471+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:50:48.476 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.471+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:50:48.476 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.471+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:50:48.476 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.471+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:50:48.477 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.471+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:50:48.477 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.471+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:50:48.477 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.471+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:50:48.477 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.471+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:50:48.477 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:48.471+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:50:48.582 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:48.575+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:50:48.582 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:48.575+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:50:48.582 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:48.575+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:48.582 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:48.575+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:48.582 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:48.575+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:50:48.582 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:48.575+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:48.582 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:48.575+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:50:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:50:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:50:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:50:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_46 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 journal 128M 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_46 128M 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_46 2026-02-19T20:50:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:50:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:50:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:50:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:50:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:50:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_46 2026-02-19T20:50:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T20:50:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:50:48.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=11 2026-02-19T20:50:48.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:50:48.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:50:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 13021184 bytes 2026-02-19T20:50:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 34924544 bytes 2026-02-19T20:50:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 50576384 bytes 2026-02-19T20:50:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 83326976 bytes 2026-02-19T20:50:51.081 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:51.075+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:50:51.083 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:51.079+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:51.084 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:51.079+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc820 2026-02-19T20:50:51.084 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:51.079+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:51.084 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:51.079+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:51.084 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:51.079+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:51.084 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:51.079+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 106286080 bytes 2026-02-19T20:50:52.362 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:52.355+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:52.405 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:52.399+0000 7fd63ae12640 -1 reset not still connected to 0x55f112563860 2026-02-19T20:50:52.405 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:52.399+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a21a0 2026-02-19T20:50:52.405 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:50:52.399+0000 7fd63ae12640 -1 reset not still connected to 0x55f1202a2680 2026-02-19T20:50:52.529 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:52.523+0000 7f93a8cad640 -1 reset not still connected to 0x556e31eba270 2026-02-19T20:50:52.530 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:52.523+0000 7f93a8cad640 -1 reset not still connected to 0x556e32496a90 2026-02-19T20:50:52.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 157501440 bytes 2026-02-19T20:50:53.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 223414272 bytes 2026-02-19T20:50:53.882 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:53.875+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:50:53.884 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:53.879+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:50:54.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 311497728 bytes 2026-02-19T20:50:55.259 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.255+0000 7f93a8cad640 -1 reset not still connected to 0x556e493dfad0 2026-02-19T20:50:55.259 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.255+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:50:55.260 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.255+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:50:55.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 348222464 bytes 2026-02-19T20:50:55.318 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.311+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb2a90 2026-02-19T20:50:55.318 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.311+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb64410 2026-02-19T20:50:55.318 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.311+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb65930 2026-02-19T20:50:55.318 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.311+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d3301a0 2026-02-19T20:50:55.318 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.311+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d995f0 2026-02-19T20:50:55.318 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.311+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2680 2026-02-19T20:50:55.318 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.311+0000 7f93a8cad640 -1 reset not still connected to 0x556e493dfad0 2026-02-19T20:50:55.318 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.311+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:50:55.318 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.311+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:50:55.496 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.491+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:50:55.497 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.491+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:55.497 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.491+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:55.499 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.495+0000 7f93a8cad640 -1 reset not still connected to 0x556e747ab6c0 2026-02-19T20:50:55.520 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:50:55.520 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:55.520 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:55.520 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.515+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:55.522 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.519+0000 7f93a8cad640 -1 reset not still connected to 0x556e747ab6c0 2026-02-19T20:50:55.808 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.803+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:50:55.809 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.803+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:50:55.809 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.803+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:50:55.809 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.803+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:50:55.810 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:55.803+0000 7f93a8cad640 -1 reset not still connected to 0x556e747ab6c0 2026-02-19T20:50:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 429415424 bytes 2026-02-19T20:50:56.645 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:56.626+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:50:56.645 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:56.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:50:56.646 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:56.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:50:56.646 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:56.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:50:56.646 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:56.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:50:56.646 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:50:56.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:50:57.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 470733824 bytes 2026-02-19T20:50:57.989 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:57.982+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:50:57.990 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:57.982+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:50:57.990 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:57.982+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:50:57.990 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:57.982+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:50:57.991 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:57.986+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:50:58.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 480352256 bytes 2026-02-19T20:50:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 541203456 bytes 2026-02-19T20:50:59.176 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.170+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09872f70 2026-02-19T20:50:59.177 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.170+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50680 2026-02-19T20:50:59.177 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.170+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb01a0 2026-02-19T20:50:59.177 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.170+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738dd0 2026-02-19T20:50:59.178 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.170+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24147ad0 2026-02-19T20:50:59.178 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.170+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6820 2026-02-19T20:50:59.178 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.170+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e7520 2026-02-19T20:50:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 579852288 bytes 2026-02-19T20:50:59.789 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.782+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:50:59.789 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.782+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:50:59.789 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.782+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:50:59.789 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.782+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:50:59.790 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:50:59.782+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:50:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 588208128 bytes 2026-02-19T20:50:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:50:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:50:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:50:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_47 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 journal 128M 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:50:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_47 128M 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:50:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_47 2026-02-19T20:50:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:50:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:50:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:50:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:50:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:50:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_47 2026-02-19T20:50:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:50:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:50:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T20:50:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:50:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=7 2026-02-19T20:50:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:50:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:51:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 44558336 bytes 2026-02-19T20:51:00.963 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:00.958+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:51:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 52575232 bytes 2026-02-19T20:51:01.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 87299072 bytes 2026-02-19T20:51:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 121877504 bytes 2026-02-19T20:51:02.163 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:02.158+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b24f70 2026-02-19T20:51:02.164 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:02.158+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b25e10 2026-02-19T20:51:02.918 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:02.910+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:51:02.919 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:02.914+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:51:03.027 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:03.022+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6820 2026-02-19T20:51:03.028 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:03.022+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e7520 2026-02-19T20:51:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 178438144 bytes 2026-02-19T20:51:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 269547520 bytes 2026-02-19T20:51:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 317167616 bytes 2026-02-19T20:51:05.324 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:05.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2f520 2026-02-19T20:51:05.324 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:05.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:51:05.324 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:05.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:51:05.324 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:05.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:51:05.325 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:05.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:51:05.325 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:05.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:51:05.325 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:05.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:51:05.325 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:05.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:51:05.325 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:05.318+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:51:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 357820416 bytes 2026-02-19T20:51:06.266 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:06.258+0000 7fd63ae12640 -1 reset not still connected to 0x55f111349860 2026-02-19T20:51:06.266 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:06.258+0000 7fd63ae12640 -1 reset not still connected to 0x55f111fd1ba0 2026-02-19T20:51:06.266 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:06.258+0000 7fd63ae12640 -1 reset not still connected to 0x55f1125632b0 2026-02-19T20:51:06.267 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:06.258+0000 7fd63ae12640 -1 reset not still connected to 0x55f1174ca9c0 2026-02-19T20:51:06.267 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:06.258+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d944e0 2026-02-19T20:51:06.267 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:06.258+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d95ee0 2026-02-19T20:51:06.267 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:06.258+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b24f70 2026-02-19T20:51:06.267 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:06.258+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b25e10 2026-02-19T20:51:06.421 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.414+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09872f70 2026-02-19T20:51:06.422 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.414+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50680 2026-02-19T20:51:06.423 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.418+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb01a0 2026-02-19T20:51:06.423 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.418+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738dd0 2026-02-19T20:51:06.423 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.418+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24147ad0 2026-02-19T20:51:06.423 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.418+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6820 2026-02-19T20:51:06.423 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.418+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e7520 2026-02-19T20:51:06.720 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.714+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f50680 2026-02-19T20:51:06.721 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.714+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb01a0 2026-02-19T20:51:06.721 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.714+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738dd0 2026-02-19T20:51:06.721 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.714+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24147ad0 2026-02-19T20:51:06.721 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.714+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6820 2026-02-19T20:51:06.722 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.714+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e7520 2026-02-19T20:51:06.824 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.818+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b040 2026-02-19T20:51:06.827 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.822+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:51:06.828 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.822+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:51:06.828 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.822+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:51:06.828 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:06.822+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:51:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_48 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 journal 128M 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:06.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_48 128M 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:51:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:51:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:51:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_48 2026-02-19T20:51:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:51:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:51:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:51:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:51:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:51:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_48 2026-02-19T20:51:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T20:51:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:51:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=14 2026-02-19T20:51:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:51:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:51:07.135 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:07.130+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:51:07.137 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:07.130+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:51:07.138 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:07.130+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:51:07.138 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:07.130+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:51:07.455 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:07.450+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:51:07.456 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:07.450+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:51:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 50927616 bytes 2026-02-19T20:51:08.963 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:08.958+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdc410 2026-02-19T20:51:08.965 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:08.958+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:51:08.965 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:08.958+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:51:08.965 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:08.958+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:51:08.966 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:08.962+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:51:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 128104448 bytes 2026-02-19T20:51:10.013 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.006+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24147ad0 2026-02-19T20:51:10.014 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.006+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6820 2026-02-19T20:51:10.212 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.206+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:51:10.214 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.206+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:51:10.214 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.206+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:51:10.215 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.210+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:51:10.215 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.210+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1d110 2026-02-19T20:51:10.216 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.210+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:51:10.216 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.210+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:51:10.216 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.210+0000 7fe4dd68c640 -1 reset not still connected to 0x564a098badd0 2026-02-19T20:51:10.216 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.210+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:51:10.217 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.210+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:51:10.217 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.210+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:51:10.217 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.210+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:51:10.405 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.398+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:51:10.406 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.398+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1450 2026-02-19T20:51:10.406 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.398+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:51:10.406 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:10.398+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:51:10.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 172816384 bytes 2026-02-19T20:51:10.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 186437632 bytes 2026-02-19T20:51:11.415 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:11.410+0000 7fe4dd68c640 -1 reset not still connected to 0x564a1cd28680 2026-02-19T20:51:11.417 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:11.410+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f2c1a0 2026-02-19T20:51:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 234492928 bytes 2026-02-19T20:51:11.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 253407232 bytes 2026-02-19T20:51:12.669 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.662+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44820 2026-02-19T20:51:12.670 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.662+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45790 2026-02-19T20:51:12.672 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.666+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2f520 2026-02-19T20:51:12.672 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.666+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2ec30 2026-02-19T20:51:12.672 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.666+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cf5f0 2026-02-19T20:51:12.673 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.666+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfe10 2026-02-19T20:51:12.673 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.666+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3d40 2026-02-19T20:51:12.673 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.666+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aaa90 2026-02-19T20:51:12.673 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.666+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bc680 2026-02-19T20:51:12.673 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.666+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:51:12.673 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:12.666+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f9860 2026-02-19T20:51:12.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 315319296 bytes 2026-02-19T20:51:12.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 315926528 bytes 2026-02-19T20:51:13.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 327824384 bytes 2026-02-19T20:51:13.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 365732864 bytes 2026-02-19T20:51:14.146 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:14.138+0000 7f93a8cad640 -1 reset not still connected to 0x556e33719a00 2026-02-19T20:51:14.147 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:14.138+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb2a90 2026-02-19T20:51:14.147 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:14.138+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb64410 2026-02-19T20:51:14.147 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:14.138+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb65930 2026-02-19T20:51:14.147 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:14.138+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d3301a0 2026-02-19T20:51:14.147 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:14.138+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d995f0 2026-02-19T20:51:14.148 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:14.142+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2680 2026-02-19T20:51:14.148 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:14.142+0000 7f93a8cad640 -1 reset not still connected to 0x556e493dfad0 2026-02-19T20:51:14.148 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:14.142+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:51:14.148 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:14.142+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:51:14.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 417088512 bytes 2026-02-19T20:51:14.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 420275200 bytes 2026-02-19T20:51:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 466663424 bytes 2026-02-19T20:51:16.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 553522176 bytes 2026-02-19T20:51:17.164 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:17.158+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb01a0 2026-02-19T20:51:17.164 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:17.158+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738dd0 2026-02-19T20:51:17.165 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:17.158+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24147ad0 2026-02-19T20:51:17.165 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:17.158+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6820 2026-02-19T20:51:17.165 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:17.158+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e7520 2026-02-19T20:51:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 576680960 bytes 2026-02-19T20:51:18.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 625096704 bytes 2026-02-19T20:51:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 670594048 bytes 2026-02-19T20:51:19.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 689796096 bytes 2026-02-19T20:51:19.877 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T20:51:19.877 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T20:51:19.877 INFO:tasks.rbd_mirror_thrash:waiting for 0.40868395472362873 secs before reviving daemons 2026-02-19T20:51:19.878 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T20:51:19.870+0000 7fd49d67e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 81612) UID: 1000 2026-02-19T20:51:19.891 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:19.886+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bdc70 2026-02-19T20:51:20.287 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T20:51:20.287 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:51:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 758947840 bytes 2026-02-19T20:51:20.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2000 IOs, 775785472 bytes 2026-02-19T20:51:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:51:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:51:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:51:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:51:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:51:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_49 2026-02-19T20:51:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 journal 128M 2026-02-19T20:51:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T20:51:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:51:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_49 128M 2026-02-19T20:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T20:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:51:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:51:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_49 2026-02-19T20:51:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:51:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:51:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:51:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:51:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:51:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_49 2026-02-19T20:51:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T20:51:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:51:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=5 2026-02-19T20:51:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:51:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:51:21.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 25235456 bytes 2026-02-19T20:51:22.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 62441472 bytes 2026-02-19T20:51:22.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 71492608 bytes 2026-02-19T20:51:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 108184576 bytes 2026-02-19T20:51:23.866 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:23.858+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6820 2026-02-19T20:51:23.868 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:23.862+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e7520 2026-02-19T20:51:23.959 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:23.954+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6820 2026-02-19T20:51:23.961 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:23.954+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e7520 2026-02-19T20:51:24.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 113842176 bytes 2026-02-19T20:51:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 161510400 bytes 2026-02-19T20:51:25.089 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:25.082+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:51:25.090 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:25.082+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:51:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:51:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:51:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:51:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:51:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_50 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 journal 128M 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n journal ']' 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_50 128M 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 128M ']' 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=128M 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:51:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_50 2026-02-19T20:51:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ is_pool_mirror_mode_image mirror 2026-02-19T20:51:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' pool = image ']' 2026-02-19T20:51:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ case "${pool}" in 2026-02-19T20:51:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 1 2026-02-19T20:51:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n '' ']' 2026-02-19T20:51:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_50 2026-02-19T20:51:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T20:51:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T20:51:26.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=14 2026-02-19T20:51:26.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T20:51:26.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2026-02-19T20:51:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 24499200 bytes 2026-02-19T20:51:26.388 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T20:51:26.389 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T20:51:26.389 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T20:51:26.391 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T20:51:26.391 INFO:tasks.rbd_mirror_thrash:waiting for 86.34801686429749 secs before thrashing 2026-02-19T20:51:27.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 73699328 bytes 2026-02-19T20:51:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 103985152 bytes 2026-02-19T20:51:29.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 162658304 bytes 2026-02-19T20:51:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 177408000 bytes 2026-02-19T20:51:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 241650688 bytes 2026-02-19T20:51:31.825 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:31.818+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:51:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 335371264 bytes 2026-02-19T20:51:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 417718272 bytes 2026-02-19T20:51:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 475139072 bytes 2026-02-19T20:51:34.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 508815360 bytes 2026-02-19T20:51:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 521314304 bytes 2026-02-19T20:51:34.918 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:34.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:51:35.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 553389056 bytes 2026-02-19T20:51:35.906 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.898+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:51:35.907 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.898+0000 7f93a8cad640 -1 reset not still connected to 0x556e33e3ec30 2026-02-19T20:51:35.907 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.898+0000 7f93a8cad640 -1 reset not still connected to 0x556e347328f0 2026-02-19T20:51:35.907 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.898+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:51:35.907 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.898+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:51:35.907 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.898+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ad0 2026-02-19T20:51:35.909 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.902+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:51:35.909 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.902+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2fee0 2026-02-19T20:51:35.909 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.902+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:51:35.909 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.902+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:51:35.909 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.902+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:51:35.909 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.902+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:51:35.909 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.902+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:51:35.910 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.902+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:51:35.910 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.902+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:51:35.910 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:35.902+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:51:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 593257472 bytes 2026-02-19T20:51:36.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 685243392 bytes 2026-02-19T20:51:36.832 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:36.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T20:51:36.834 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:36.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:51:36.834 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:36.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1da00 2026-02-19T20:51:36.834 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:36.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:51:36.834 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:36.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:51:36.834 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:36.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:51:36.834 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:36.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:51:36.834 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:36.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:51:36.834 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:36.826+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:51:37.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 725428224 bytes 2026-02-19T20:51:37.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 748292096 bytes 2026-02-19T20:51:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 773680128 bytes 2026-02-19T20:51:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 869000192 bytes 2026-02-19T20:51:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 886507520 bytes 2026-02-19T20:51:40.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T20:51:40.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T20:51:40.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T20:51:40.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:51:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_pool_images cluster2 mirror 50 2026-02-19T20:51:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_count=50 2026-02-19T20:51:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:51:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local count 2026-02-19T20:51:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_count=0 2026-02-19T20:51:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:51:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 40 2026-02-19T20:51:40.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in `seq 1 40` 2026-02-19T20:51:40.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 1 -ne 1 2026-02-19T20:51:40.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-19T20:51:40.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep 'images: ' 2026-02-19T20:51:40.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut '-d ' -f 2 2026-02-19T20:51:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: failed to query services: (13) Permission denied 2026-02-19T20:51:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ count=50 2026-02-19T20:51:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 50 = 50 2026-02-19T20:51:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:51:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_pool_healthy cluster2 mirror 2026-02-19T20:51:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:51:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:51:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:51:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state 2026-02-19T20:51:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 40 2026-02-19T20:51:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in `seq 1 40` 2026-02-19T20:51:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 1 -ne 1 2026-02-19T20:51:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-19T20:51:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep 'image health:' 2026-02-19T20:51:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut '-d ' -f 3 2026-02-19T20:51:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: failed to query services: (13) Permission denied 2026-02-19T20:51:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=WARNING 2026-02-19T20:51:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test WARNING = ERROR 2026-02-19T20:51:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test WARNING = OK 2026-02-19T20:51:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in `seq 1 40` 2026-02-19T20:51:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 -ne 1 2026-02-19T20:51:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 30 2026-02-19T20:51:40.249 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.242+0000 7f93a8cad640 -1 reset not still connected to 0x556e31eba270 2026-02-19T20:51:40.250 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.242+0000 7f93a8cad640 -1 reset not still connected to 0x556e31ebb6c0 2026-02-19T20:51:40.251 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.246+0000 7f93a8cad640 -1 reset not still connected to 0x556e32496a90 2026-02-19T20:51:40.251 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.246+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275cf70 2026-02-19T20:51:40.251 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.246+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:51:40.251 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.246+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:51:40.251 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.246+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:51:40.251 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.246+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3040 2026-02-19T20:51:40.251 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.246+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:51:40.251 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.246+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:51:40.251 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.246+0000 7f93a8cad640 -1 reset not still connected to 0x556e747ab6c0 2026-02-19T20:51:40.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:40.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:51:44.160 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:51:44.160 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:51:44.160 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:51:44.161 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec2270 2026-02-19T20:51:44.161 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:51:44.161 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:51:44.161 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f51c70 2026-02-19T20:51:44.161 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:51:44.162 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:51:44.162 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:51:44.162 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:44.154+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:51:46.635 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.626+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:51:46.636 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:51:46.637 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:51:46.637 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:51:46.637 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:51:46.637 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:51:46.637 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:51:46.638 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.630+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:51:46.796 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44750 2026-02-19T20:51:46.796 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:51:46.796 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e33e3ec30 2026-02-19T20:51:46.796 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e347328f0 2026-02-19T20:51:46.796 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb65ee0 2026-02-19T20:51:46.796 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:51:46.796 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ad0 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2fee0 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:51:46.797 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:46.790+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:51:47.917 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:47.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:51:47.918 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:47.910+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:51:47.919 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:47.914+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:51:47.919 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:47.914+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:51:47.919 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:47.914+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:51:47.920 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:47.914+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:51:47.921 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:47.914+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:51:47.922 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:47.914+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:51:48.215 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.210+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:51:48.215 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.210+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:51:48.216 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.210+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ad0 2026-02-19T20:51:48.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:51:48.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2fee0 2026-02-19T20:51:48.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:51:48.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:51:48.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:51:48.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:51:48.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:51:48.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:51:48.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:51:48.220 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:48.214+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:51:50.728 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:50.722+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275cf70 2026-02-19T20:51:50.728 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:50.722+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:51:50.728 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:50.722+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:51:54.486 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.574184598650089) > weight (0.25) 2026-02-19T20:51:54.486 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-19T20:51:54.487 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-19T20:51:54.487 INFO:tasks.rbd_mirror_thrash:waiting for 6.303671797020565 secs before reviving daemons 2026-02-19T20:51:54.487 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T20:51:54.478+0000 7fa4a110e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 80471) UID: 1000 2026-02-19T20:51:55.511 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.506+0000 7fe4dd68c640 -1 reset not still connected to 0x5649ffdd4b60 2026-02-19T20:51:55.513 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.506+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:51:55.513 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.506+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:51:55.513 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.506+0000 7fe4dd68c640 -1 reset not still connected to 0x564a004151e0 2026-02-19T20:51:55.513 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.506+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:51:55.514 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.506+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:51:55.514 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.506+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:51:55.514 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.506+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f51c70 2026-02-19T20:51:55.516 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.510+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:51:55.516 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.510+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:51:55.516 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:55.510+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:51:56.473 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:56.466+0000 7fd63ae12640 -1 reset not still connected to 0x55f11277cc30 2026-02-19T20:51:56.473 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:56.466+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d95790 2026-02-19T20:51:56.605 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:56.598+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df696c0 2026-02-19T20:51:57.636 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:57.630+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b24f70 2026-02-19T20:51:57.745 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:57.738+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b24f70 2026-02-19T20:51:57.747 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:57.742+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b25e10 2026-02-19T20:51:57.775 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:57.770+0000 7fd63ae12640 -1 reset not still connected to 0x55f10ff63520 2026-02-19T20:51:57.775 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:57.770+0000 7fd63ae12640 -1 reset not still connected to 0x55f11025fa00 2026-02-19T20:51:57.775 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:57.770+0000 7fd63ae12640 -1 reset not still connected to 0x55f111fd1ba0 2026-02-19T20:51:57.775 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:57.770+0000 7fd63ae12640 -1 reset not still connected to 0x55f1125632b0 2026-02-19T20:51:57.776 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:57.770+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d944e0 2026-02-19T20:51:57.776 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:57.770+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d95ee0 2026-02-19T20:51:57.776 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:51:57.770+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b24f70 2026-02-19T20:51:58.113 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:58.106+0000 7f93a8cad640 -1 reset not still connected to 0x556e31bc48f0 2026-02-19T20:51:58.113 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:58.106+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d110 2026-02-19T20:51:58.114 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:58.106+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:51:58.114 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:58.106+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:51:58.114 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:51:58.106+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:51:58.384 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:58.378+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:51:58.385 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:58.378+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1520 2026-02-19T20:51:58.385 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:58.378+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1ee0 2026-02-19T20:51:58.386 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:58.378+0000 7fe4dd68c640 -1 reset not still connected to 0x564a117392b0 2026-02-19T20:51:58.386 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:58.378+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11739790 2026-02-19T20:51:58.386 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:58.378+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042dd0 2026-02-19T20:51:58.386 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:58.378+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34ea0 2026-02-19T20:51:58.386 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:58.378+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df696c0 2026-02-19T20:51:59.300 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:59.294+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042dd0 2026-02-19T20:51:59.301 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:51:59.294+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df696c0 2026-02-19T20:52:00.659 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.654+0000 7f93a8cad640 -1 reset not still connected to 0x556e31bc48f0 2026-02-19T20:52:00.660 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.654+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d110 2026-02-19T20:52:00.663 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:52:00.664 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e33e3ec30 2026-02-19T20:52:00.664 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e347328f0 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb65ee0 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ad0 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2fee0 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:00.665 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:52:00.666 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:00.658+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:00.791 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-19T20:52:00.791 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:52:00.792 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-19T20:52:00.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-19T20:52:00.792 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-19T20:52:00.794 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-19T20:52:00.794 INFO:tasks.rbd_mirror_thrash:waiting for 84.96692172988428 secs before thrashing 2026-02-19T20:52:01.427 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:01.418+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:52:01.427 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:01.418+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:52:01.427 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:01.418+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f51c70 2026-02-19T20:52:01.429 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:01.422+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:52:01.429 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:01.422+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:52:01.429 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:01.422+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:52:01.429 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:01.422+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:52:03.203 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.193+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:52:03.203 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.197+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e820 2026-02-19T20:52:03.204 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.197+0000 7fe4dd68c640 -1 reset not still connected to 0x564a004151e0 2026-02-19T20:52:03.205 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.197+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:52:03.205 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.197+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:52:03.205 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.197+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:52:03.205 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.197+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f51c70 2026-02-19T20:52:03.207 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.197+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:52:03.207 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.197+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:52:03.207 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.197+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:52:03.207 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.197+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:52:03.976 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.969+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:52:03.976 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.969+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f51c70 2026-02-19T20:52:03.976 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.969+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:52:03.976 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.969+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:52:03.977 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.969+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:52:03.977 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:03.969+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:52:04.509 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:04.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a030676c0 2026-02-19T20:52:04.510 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:04.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:52:04.510 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:04.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1520 2026-02-19T20:52:04.510 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:04.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a117392b0 2026-02-19T20:52:04.512 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:04.505+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11739790 2026-02-19T20:52:04.512 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:04.505+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042dd0 2026-02-19T20:52:04.512 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:04.505+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34ea0 2026-02-19T20:52:04.512 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:04.505+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df696c0 2026-02-19T20:52:05.505 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:05.497+0000 7fe4dd68c640 -1 reset not still connected to 0x564a030676c0 2026-02-19T20:52:05.508 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:05.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:52:05.508 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:05.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1520 2026-02-19T20:52:05.510 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:05.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1ee0 2026-02-19T20:52:05.510 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:05.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a117392b0 2026-02-19T20:52:05.510 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:05.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11739790 2026-02-19T20:52:05.510 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:05.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042dd0 2026-02-19T20:52:05.510 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:05.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34ea0 2026-02-19T20:52:05.510 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:05.501+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df696c0 2026-02-19T20:52:06.815 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:06.805+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:52:06.815 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:06.805+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1520 2026-02-19T20:52:06.816 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:06.805+0000 7fe4dd68c640 -1 reset not still connected to 0x564a117392b0 2026-02-19T20:52:06.816 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:06.809+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11739790 2026-02-19T20:52:06.816 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:06.809+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042dd0 2026-02-19T20:52:06.816 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:06.809+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34ea0 2026-02-19T20:52:06.817 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:06.809+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df696c0 2026-02-19T20:52:08.256 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:08.249+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:52:08.992 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:08.985+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T20:52:08.992 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:08.985+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1da00 2026-02-19T20:52:08.992 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:08.985+0000 7fe4dd68c640 -1 reset not still connected to 0x564a06fd8410 2026-02-19T20:52:08.993 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:08.985+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T20:52:08.993 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:08.985+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T20:52:08.993 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:08.985+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f340d0 2026-02-19T20:52:08.993 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:08.985+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35860 2026-02-19T20:52:08.994 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:08.985+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f35e10 2026-02-19T20:52:09.910 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.901+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:52:09.911 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.901+0000 7f93a8cad640 -1 reset not still connected to 0x556e33e3ec30 2026-02-19T20:52:09.911 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.901+0000 7f93a8cad640 -1 reset not still connected to 0x556e347328f0 2026-02-19T20:52:09.912 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb65ee0 2026-02-19T20:52:09.912 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:52:09.912 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:52:09.912 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ad0 2026-02-19T20:52:09.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:52:09.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2fee0 2026-02-19T20:52:09.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:52:09.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:52:09.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:09.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:52:09.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:52:09.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:09.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:52:09.913 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:09.905+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-19T20:52:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep 'image health:' 2026-02-19T20:52:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut '-d ' -f 3 2026-02-19T20:52:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: failed to query services: (13) Permission denied 2026-02-19T20:52:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=OK 2026-02-19T20:52:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test OK = ERROR 2026-02-19T20:52:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test OK = OK 2026-02-19T20:52:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_pool_images cluster1 mirror 50 2026-02-19T20:52:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_count=50 2026-02-19T20:52:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:52:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local count 2026-02-19T20:52:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_count=0 2026-02-19T20:52:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:52:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 40 2026-02-19T20:52:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in `seq 1 40` 2026-02-19T20:52:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 1 -ne 1 2026-02-19T20:52:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-19T20:52:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep 'images: ' 2026-02-19T20:52:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut '-d ' -f 2 2026-02-19T20:52:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: failed to query services: (13) Permission denied 2026-02-19T20:52:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ count=50 2026-02-19T20:52:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 50 = 50 2026-02-19T20:52:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_pool_healthy cluster1 mirror 2026-02-19T20:52:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:52:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state 2026-02-19T20:52:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 40 2026-02-19T20:52:10.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in `seq 1 40` 2026-02-19T20:52:10.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 1 -ne 1 2026-02-19T20:52:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-19T20:52:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep 'image health:' 2026-02-19T20:52:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut '-d ' -f 3 2026-02-19T20:52:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: failed to query services: (13) Permission denied 2026-02-19T20:52:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=WARNING 2026-02-19T20:52:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test WARNING = ERROR 2026-02-19T20:52:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test WARNING = OK 2026-02-19T20:52:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in `seq 1 40` 2026-02-19T20:52:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 -ne 1 2026-02-19T20:52:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 30 2026-02-19T20:52:10.994 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:10.985+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:10.995 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:10.989+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:52:10.996 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:10.989+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:52:10.997 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:10.989+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:10.997 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:10.989+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:52:10.997 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:10.989+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:11.646 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:11.637+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:52:11.646 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:11.637+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:52:11.666 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.657+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0ac8b860 2026-02-19T20:52:11.668 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.661+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1520 2026-02-19T20:52:11.669 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.661+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0deb1ee0 2026-02-19T20:52:11.669 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.661+0000 7fe4dd68c640 -1 reset not still connected to 0x564a117392b0 2026-02-19T20:52:11.669 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.661+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11739790 2026-02-19T20:52:11.669 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.661+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042dd0 2026-02-19T20:52:11.669 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.661+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34ea0 2026-02-19T20:52:11.669 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.661+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df696c0 2026-02-19T20:52:11.823 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec2270 2026-02-19T20:52:11.823 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:52:11.823 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:52:11.823 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f51c70 2026-02-19T20:52:11.823 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:52:11.824 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.813+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:52:11.824 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.817+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:52:11.824 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:11.817+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:52:12.286 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.277+0000 7fe4dd68c640 -1 reset not still connected to 0x564a117392b0 2026-02-19T20:52:12.287 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.281+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11739790 2026-02-19T20:52:12.289 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.281+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042dd0 2026-02-19T20:52:12.289 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.281+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34ea0 2026-02-19T20:52:12.290 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.281+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df696c0 2026-02-19T20:52:12.292 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e5b0 2026-02-19T20:52:12.293 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0031e680 2026-02-19T20:52:12.293 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a004151e0 2026-02-19T20:52:12.294 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec2270 2026-02-19T20:52:12.294 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:52:12.294 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:52:12.294 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f51c70 2026-02-19T20:52:12.294 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:52:12.295 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.285+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:52:12.295 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.289+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:52:12.295 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:12.289+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:52:12.787 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:12.777+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f8680 2026-02-19T20:52:13.075 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:13.069+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:52:13.954 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:13.945+0000 7f93a8cad640 -1 reset not still connected to 0x556e33718680 2026-02-19T20:52:13.955 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:13.945+0000 7f93a8cad640 -1 reset not still connected to 0x556e33bb2a90 2026-02-19T20:52:13.955 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:13.945+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb64410 2026-02-19T20:52:13.955 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:13.945+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d3301a0 2026-02-19T20:52:13.955 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:13.945+0000 7f93a8cad640 -1 reset not still connected to 0x556e44d995f0 2026-02-19T20:52:13.955 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:13.945+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2680 2026-02-19T20:52:13.955 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:13.945+0000 7f93a8cad640 -1 reset not still connected to 0x556e493dfad0 2026-02-19T20:52:13.956 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:13.945+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a3960d0 2026-02-19T20:52:13.956 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:13.949+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e3e10 2026-02-19T20:52:13.956 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:13.949+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:52:14.702 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:14.693+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f89c0 2026-02-19T20:52:14.953 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:14.945+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f8680 2026-02-19T20:52:15.469 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.461+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:15.470 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.461+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:15.767 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.757+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:52:15.768 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e33e3ec30 2026-02-19T20:52:15.770 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e347328f0 2026-02-19T20:52:15.770 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb65ee0 2026-02-19T20:52:15.770 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:52:15.770 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:52:15.770 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ad0 2026-02-19T20:52:15.771 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:52:15.771 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2fee0 2026-02-19T20:52:15.771 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:52:15.771 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:52:15.771 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:15.771 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:52:15.772 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:52:15.772 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:15.772 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:52:15.772 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:15.761+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:17.046 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:17.037+0000 7fe4dd68c640 -1 reset not still connected to 0x564a004151e0 2026-02-19T20:52:17.048 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:17.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec2270 2026-02-19T20:52:17.048 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:17.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:52:17.049 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:17.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:52:17.049 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:17.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f51c70 2026-02-19T20:52:17.049 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:17.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:52:17.050 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:17.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:52:17.050 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:17.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:52:17.050 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:17.041+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:52:17.327 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:17.321+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:17.330 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:17.321+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:19.305 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.297+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:52:19.307 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.297+0000 7f93a8cad640 -1 reset not still connected to 0x556e33e3ec30 2026-02-19T20:52:19.307 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.297+0000 7f93a8cad640 -1 reset not still connected to 0x556e347328f0 2026-02-19T20:52:19.307 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.297+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:52:19.307 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.297+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:52:19.307 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.297+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ad0 2026-02-19T20:52:19.308 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:52:19.308 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2fee0 2026-02-19T20:52:19.308 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:52:19.308 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:52:19.309 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:19.309 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:52:19.309 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:52:19.309 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:19.309 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:52:19.309 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:19.301+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:19.619 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:19.613+0000 7fe4dd68c640 -1 reset not still connected to 0x564a117392b0 2026-02-19T20:52:19.620 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:19.613+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11739790 2026-02-19T20:52:19.621 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:19.613+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042dd0 2026-02-19T20:52:19.621 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:19.613+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f34ea0 2026-02-19T20:52:19.622 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:19.613+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df696c0 2026-02-19T20:52:20.227 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e2fbffc70 2026-02-19T20:52:20.227 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e31bc48f0 2026-02-19T20:52:20.227 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e32496ea0 2026-02-19T20:52:20.227 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d110 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c44750 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e33e3ec30 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e347328f0 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb65ee0 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ad0 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2fee0 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:52:20.228 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:52:20.229 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:20.229 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:52:20.229 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:52:20.229 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:20.229 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:52:20.229 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:20.217+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:21.331 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:21.321+0000 7fe4dd68c640 -1 reset not still connected to 0x564a004151e0 2026-02-19T20:52:21.333 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:21.325+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00b8d860 2026-02-19T20:52:21.334 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:21.325+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00ec2270 2026-02-19T20:52:21.334 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:21.325+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f3b5f0 2026-02-19T20:52:21.334 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:21.325+0000 7fe4dd68c640 -1 reset not still connected to 0x564a03067e10 2026-02-19T20:52:21.334 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:21.325+0000 7fe4dd68c640 -1 reset not still connected to 0x564a09f51c70 2026-02-19T20:52:21.334 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:21.325+0000 7fe4dd68c640 -1 reset not still connected to 0x564a0abdd790 2026-02-19T20:52:21.334 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:21.325+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12042c30 2026-02-19T20:52:21.334 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:21.325+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e6f70 2026-02-19T20:52:21.334 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T20:52:21.325+0000 7fe4dd68c640 -1 reset not still connected to 0x564a3df69c70 2026-02-19T20:52:23.702 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:23.693+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f01fba0 2026-02-19T20:52:23.702 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:23.693+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e35f0 2026-02-19T20:52:23.702 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:23.693+0000 7f93a8cad640 -1 reset not still connected to 0x556e727f9860 2026-02-19T20:52:23.702 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:23.693+0000 7f93a8cad640 -1 reset not still connected to 0x556e748bcf70 2026-02-19T20:52:23.703 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:23.693+0000 7f93a8cad640 -1 reset not still connected to 0x556e748f8680 2026-02-19T20:52:26.644 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:26.637+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:26.645 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:26.637+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:52:26.646 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:26.637+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:52:26.646 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:26.637+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:26.646 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:26.637+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:52:26.646 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:26.637+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:28.189 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.181+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:52:28.189 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.181+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:52:28.190 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.181+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ad0 2026-02-19T20:52:28.191 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.181+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:52:28.191 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.181+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2fee0 2026-02-19T20:52:28.191 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.181+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:52:28.191 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.181+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:52:28.192 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.185+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:28.192 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.185+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:52:28.192 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.185+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:52:28.192 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.185+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:28.192 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.185+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:52:28.192 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:28.185+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:30.412 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:52:30.405+0000 7fd63ae12640 -1 reset not still connected to 0x55f11025f6c0 2026-02-19T20:52:30.413 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:52:30.405+0000 7fd63ae12640 -1 reset not still connected to 0x55f11025fa00 2026-02-19T20:52:30.414 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:52:30.405+0000 7fd63ae12640 -1 reset not still connected to 0x55f111349860 2026-02-19T20:52:30.414 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:52:30.405+0000 7fd63ae12640 -1 reset not still connected to 0x55f111fd1ba0 2026-02-19T20:52:30.414 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:52:30.405+0000 7fd63ae12640 -1 reset not still connected to 0x55f1125632b0 2026-02-19T20:52:30.414 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:52:30.405+0000 7fd63ae12640 -1 reset not still connected to 0x55f1174ca9c0 2026-02-19T20:52:30.414 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:52:30.405+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d944e0 2026-02-19T20:52:30.415 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:52:30.405+0000 7fd63ae12640 -1 reset not still connected to 0x55f125d95ee0 2026-02-19T20:52:30.415 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:52:30.405+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b24f70 2026-02-19T20:52:30.415 INFO:tasks.ceph.cluster1.osd.1.trial013.stderr:2026-02-19T20:52:30.405+0000 7fd63ae12640 -1 reset not still connected to 0x55f126b25e10 2026-02-19T20:52:33.658 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.649+0000 7f93a8cad640 -1 reset not still connected to 0x556e31eba270 2026-02-19T20:52:33.658 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.649+0000 7f93a8cad640 -1 reset not still connected to 0x556e31ebb6c0 2026-02-19T20:52:33.660 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.653+0000 7f93a8cad640 -1 reset not still connected to 0x556e32496a90 2026-02-19T20:52:33.661 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.653+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275cf70 2026-02-19T20:52:33.661 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.653+0000 7f93a8cad640 -1 reset not still connected to 0x556e3275d380 2026-02-19T20:52:33.661 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.653+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2e9c0 2026-02-19T20:52:33.661 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.653+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2f5f0 2026-02-19T20:52:33.661 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.653+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3040 2026-02-19T20:52:33.661 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.653+0000 7f93a8cad640 -1 reset not still connected to 0x556e679e2d00 2026-02-19T20:52:33.661 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.653+0000 7f93a8cad640 -1 reset not still connected to 0x556e67a116c0 2026-02-19T20:52:33.662 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:33.653+0000 7f93a8cad640 -1 reset not still connected to 0x556e747ab6c0 2026-02-19T20:52:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-19T20:52:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep 'image health:' 2026-02-19T20:52:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut '-d ' -f 3 2026-02-19T20:52:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: failed to query services: (13) Permission denied 2026-02-19T20:52:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=OK 2026-02-19T20:52:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test OK = ERROR 2026-02-19T20:52:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test OK = OK 2026-02-19T20:52:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ testlog 'TEST: compare many images' 2026-02-19T20:52:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ tee -a /tmp/tmp.xpxdoJdHT7/rbd-mirror.test.log 2026-02-19T20:52:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ date '+%F %T' 2026-02-19T20:52:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo 2026-02-19 20:52:40 TEST: compare many images 2026-02-19T20:52:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:2026-02-19 20:52:40 TEST: compare many images 2026-02-19T20:52:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 50 2026-02-19T20:52:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:52:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_1 2026-02-19T20:52:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_1 snap 2026-02-19T20:52:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:52:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:52:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:52:42.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2026-02-19T20:52:42.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2026-02-19T20:52:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:52:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:52:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:52:43.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2026-02-19T20:52:43.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:43.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:43.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:52:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:52:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:52:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2026-02-19T20:52:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:52:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:52:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:52:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:52:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:52:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:52:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:52:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2026-02-19T20:52:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:52:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:52:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:52:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:52:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:52:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:52:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:52:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:52:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:52:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2026-02-19T20:52:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:52:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:52:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:52:43.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:52:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:52:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:52:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:52:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_1 2026-02-19T20:52:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:52:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:52:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:52:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:52:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:52:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_1 2026-02-19T20:52:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:52:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:52:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:52:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:52:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:52:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:52:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:52:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:52:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_1 2026-02-19T20:52:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_1 ']' 2026-02-19T20:52:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-19T20:52:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-19T20:52:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:44.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:44.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:44.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:44.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:44.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:44.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:44.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:44.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-19T20:52:44.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:44.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:44.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:44.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:44.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:44.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-19T20:52:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-19T20:52:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-19T20:52:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:44.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-19T20:52:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-19T20:52:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=52702' '!=' '' 2026-02-19T20:52:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:52:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=52702' = '' 2026-02-19T20:52:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:44.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:52:44.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:52:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:52:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:52:44.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:52:44.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:44.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:52:44.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:52:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:52:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:52:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:52:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=52702 2026-02-19T20:52:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 52702 2026-02-19T20:52:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:52:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_1 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_1 ']' 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-19T20:52:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:44.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-19T20:52:44.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:44.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-19T20:52:44.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:44.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:44.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:44.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-19T20:52:44.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:44.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-19T20:52:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-19T20:52:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:44.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:44.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:44.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:44.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=52702' '!=' 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_1 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_1 ']' 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-19T20:52:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-19T20:52:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=52702' '!=' 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:52:46.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_1 2026-02-19T20:52:46.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:46.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:46.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:46.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:46.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_1 ']' 2026-02-19T20:52:46.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-19T20:52:46.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-19T20:52:46.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:46.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:46.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:46.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:46.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:46.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-19T20:52:46.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:46.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:46.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:46.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:46.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:46.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:46.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:46.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:46.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:46.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:46.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:46.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:46.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:46.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:46.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:46.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-19T20:52:46.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-19T20:52:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-19T20:52:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:46.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:46.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:46.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:46.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:46.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:46.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:46.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-19T20:52:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-19T20:52:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:46.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=52702' '!=' 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:46.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:46.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:52:46.864 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:46.857+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:52:46.864 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:46.857+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:46.865 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:46.857+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:52:46.865 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:46.857+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:52:46.865 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:46.857+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:46.867 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:46.861+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:52:46.867 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:46.861+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_1 2026-02-19T20:52:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_1 ']' 2026-02-19T20:52:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-19T20:52:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-19T20:52:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:47.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:47.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-19T20:52:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-19T20:52:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-19T20:52:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:47.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:47.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:47.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:47.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:47.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-19T20:52:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-19T20:52:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=52702' '!=' 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:52:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_1 2026-02-19T20:52:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_1 ']' 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-19T20:52:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:49.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:49.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-19T20:52:50.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-19T20:52:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-19T20:52:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-19T20:52:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-19T20:52:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:50.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=52702], [object_number=49, tag_tid=1, entry_tid=52701], [object_number=48, tag_tid=1, entry_tid=52700], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:50.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=52702' '!=' 'object_number=50, tag_tid=1, entry_tid=52702' 2026-02-19T20:52:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:52:51.526 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.517+0000 7f93a8cad640 -1 reset not still connected to 0x556e32c45520 2026-02-19T20:52:51.528 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e33e3ec30 2026-02-19T20:52:51.528 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e347328f0 2026-02-19T20:52:51.529 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e3cb65ee0 2026-02-19T20:52:51.529 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e3d331c70 2026-02-19T20:52:51.529 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:52:51.529 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ad0 2026-02-19T20:52:51.529 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c9ee0 2026-02-19T20:52:51.530 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e3fb2fee0 2026-02-19T20:52:51.530 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:52:51.530 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e44f2fd40 2026-02-19T20:52:51.530 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:51.530 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d3450 2026-02-19T20:52:51.530 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e493cfba0 2026-02-19T20:52:51.530 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:51.530 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e5f78ba00 2026-02-19T20:52:51.530 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:51.521+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:52.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_1 2026-02-19T20:52:52.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:52.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:52.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:52.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:52.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_1 ']' 2026-02-19T20:52:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-19T20:52:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-19T20:52:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-19T20:52:52.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-19T20:52:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-19T20:52:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:52.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:52.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:52.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:52.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-19T20:52:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-19T20:52:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T20:52:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-19T20:52:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-19T20:52:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:52.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:52.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-19T20:52:52.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:52:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:52:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=52702], [object_number=51, tag_tid=1, entry_tid=52699]]], state=connected] 2026-02-19T20:52:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.status 2026-02-19T20:52:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:52:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2026-02-19T20:52:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:52:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:52:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:52:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:52:52.740 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T20:52:52.741 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T20:52:52.741 INFO:tasks.rbd_mirror_thrash:waiting for 6.8354331054163096 secs before reviving daemons 2026-02-19T20:52:52.741 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T20:52:52.730+0000 7f75f4322640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 84338) UID: 1000 2026-02-19T20:52:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_1': 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11167317349a 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11167317349a 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:43:27 2026 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:43:27 2026 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:44:28 2026 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11167317349a 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 8d0cbc7c-5771-4875-b666-ac016da040df 2026-02-19T20:52:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:52:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_1 snap 2026-02-19T20:52:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T20:52:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:52:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:52:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.export 2026-02-19T20:52:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_1.export 2026-02-19T20:52:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_1.export 2026-02-19T20:52:53.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.export 2026-02-19T20:52:53.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:52:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_1.export 2026-02-19T20:52:53.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:52:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_1.export 2026-02-19T20:52:53.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_1.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_1.export 2026-02-19T20:52:53.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:53.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:52:53.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_2 2026-02-19T20:52:53.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_2 snap 2026-02-19T20:52:53.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:52:53.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:53.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:52:53.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:52:53.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:52:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2026-02-19T20:52:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:52:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2026-02-19T20:52:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:52:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:52:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:52:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:52:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2026-02-19T20:52:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:52:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:52:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:52:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:52:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:52:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:52:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:52:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:52:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:52:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:52:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:52:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:56.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:56.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:56.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:56.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2026-02-19T20:52:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:52:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2 2026-02-19T20:52:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:52:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:52:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:52:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:52:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:52:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:52:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:52:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2 2026-02-19T20:52:56.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 4 closest matches: 2026-02-19T20:52:56.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_21 2026-02-19T20:52:56.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_25 2026-02-19T20:52:56.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_27 2026-02-19T20:52:56.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_29 2026-02-19T20:52:56.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:52:56.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_2 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:52:56.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:52:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:52:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:52:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:52:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:52:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:52:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:52:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:52:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:52:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_2 2026-02-19T20:52:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:52:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:52:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:52:57.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:52:57.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:52:57.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_2 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_2 2026-02-19T20:52:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:52:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:52:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:52:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:52:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:52:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:52:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:52:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:52:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:52:57.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_2 2026-02-19T20:52:57.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:52:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_2 ']' 2026-02-19T20:52:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-19T20:52:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-19T20:52:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:57.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:57.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:57.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:57.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:52:57.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:57.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-19T20:52:57.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:52:57.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:57.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:57.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:52:57.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:52:57.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:52:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:52:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:52:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:52:57.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:57.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:57.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:57.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 4 closest matches: 2026-02-19T20:52:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_21 2026-02-19T20:52:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_25 2026-02-19T20:52:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_27 2026-02-19T20:52:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_29 2026-02-19T20:52:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:52:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-19T20:52:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:52:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:52:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:52:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:52:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:57.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:57.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:57.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:57.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-19T20:52:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-19T20:52:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:52:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:52:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:52:57.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:52:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:52:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40898], [object_number=37, tag_tid=1, entry_tid=40897], [object_number=36, tag_tid=1, entry_tid=40896], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:52:57.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-19T20:52:57.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:57.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:57.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-19T20:52:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:52:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:52:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:52:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:52:57.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:52:57.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:57.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:57.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40898], [object_number=37, tag_tid=1, entry_tid=40897], [object_number=36, tag_tid=1, entry_tid=40896], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40898' '!=' '' 2026-02-19T20:52:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:52:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40898' = '' 2026-02-19T20:52:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:52:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:52:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:52:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:52:57.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:57.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:52:57.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:52:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:52:57.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:52:57.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:52:57.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:52:57.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:52:57.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:57.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:52:57.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:52:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40898 2026-02-19T20:52:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 40898 2026-02-19T20:52:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:52:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:57.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:52:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_2 2026-02-19T20:52:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_2 ']' 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:57.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:57.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:57.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:57.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:57.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:57.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:57.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:57.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:57.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:57.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:52:57.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-19T20:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:52:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:52:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:52:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:52:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:52:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:52:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:57.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:57.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:57.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 4 closest matches: 2026-02-19T20:52:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_21 2026-02-19T20:52:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_25 2026-02-19T20:52:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_27 2026-02-19T20:52:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_29 2026-02-19T20:52:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:52:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-19T20:52:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:52:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:52:57.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:52:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:52:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:52:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:57.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:52:57.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:52:57.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:52:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:52:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40898], [object_number=37, tag_tid=1, entry_tid=40897], [object_number=36, tag_tid=1, entry_tid=40896], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:57.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:52:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-19T20:52:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-19T20:52:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:57.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:52:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:52:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:52:57.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:52:57.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:52:57.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:57.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:57.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40898], [object_number=37, tag_tid=1, entry_tid=40897], [object_number=36, tag_tid=1, entry_tid=40896], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40898' '!=' 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:52:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_2 2026-02-19T20:52:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:52:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_2 ']' 2026-02-19T20:52:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-19T20:52:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-19T20:52:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:58.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:58.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:58.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:58.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:58.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:58.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-19T20:52:58.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:58.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:58.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:58.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:58.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:58.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:58.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:58.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:58.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:58.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:52:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-19T20:52:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:52:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:52:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:52:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:52:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:52:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:58.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:58.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 4 closest matches: 2026-02-19T20:52:58.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_21 2026-02-19T20:52:58.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_25 2026-02-19T20:52:58.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_27 2026-02-19T20:52:58.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_29 2026-02-19T20:52:58.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:52:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-19T20:52:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:52:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:52:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:52:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:52:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:52:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:52:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:58.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:58.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-19T20:52:58.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:58.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:58.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:58.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-19T20:52:58.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:58.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:58.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:58.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:58.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:52:58.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:52:58.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:58.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:58.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:52:58.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:52:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:52:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40898], [object_number=37, tag_tid=1, entry_tid=40897], [object_number=36, tag_tid=1, entry_tid=40896], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:52:58.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-19T20:52:58.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:58.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:58.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:58.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-19T20:52:58.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:58.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:52:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:52:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:52:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:58.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:52:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:52:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40898], [object_number=37, tag_tid=1, entry_tid=40897], [object_number=36, tag_tid=1, entry_tid=40896], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40898' '!=' 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:52:59.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_2 2026-02-19T20:52:59.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:59.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:52:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:52:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:52:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_2 ']' 2026-02-19T20:52:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-19T20:52:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-19T20:52:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:52:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:52:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:52:59.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:52:59.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:52:59.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:52:59.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:52:59.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:52:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:52:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:52:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:52:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:52:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:52:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:52:59.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:59.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:52:59.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:52:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:52:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:52:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:52:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:52:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:52:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:52:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:59.577 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T20:52:59.577 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:52:59.578 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T20:52:59.578 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T20:52:59.578 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T20:52:59.579 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T20:52:59.579 INFO:tasks.rbd_mirror_thrash:waiting for 85.23160065677823 secs before thrashing 2026-02-19T20:52:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 4 closest matches: 2026-02-19T20:52:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_21 2026-02-19T20:52:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_25 2026-02-19T20:52:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_27 2026-02-19T20:52:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_29 2026-02-19T20:52:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:52:59.586 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:59.577+0000 7f93a8cad640 -1 reset not still connected to 0x556e3e9c88f0 2026-02-19T20:52:59.586 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:59.577+0000 7f93a8cad640 -1 reset not still connected to 0x556e44b768f0 2026-02-19T20:52:59.586 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:59.577+0000 7f93a8cad640 -1 reset not still connected to 0x556e456d2750 2026-02-19T20:52:59.586 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:59.577+0000 7f93a8cad640 -1 reset not still connected to 0x556e5a396b60 2026-02-19T20:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-19T20:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:52:59.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:52:59.588 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:52:59.581+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aa820 2026-02-19T20:52:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:52:59.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:59.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:59.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:52:59.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-19T20:52:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-19T20:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-19T20:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:52:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:52:59.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:52:59.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:59.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:59.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:52:59.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:52:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:52:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40898], [object_number=37, tag_tid=1, entry_tid=40897], [object_number=36, tag_tid=1, entry_tid=40896], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:52:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-19T20:52:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-19T20:52:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:52:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:52:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:52:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:52:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:52:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:52:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:52:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:52:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:52:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:52:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:52:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40898], [object_number=37, tag_tid=1, entry_tid=40897], [object_number=36, tag_tid=1, entry_tid=40896], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:52:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:52:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40898' '!=' 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:52:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:52:59.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:53:01.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_2 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_2 ']' 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-19T20:53:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-19T20:53:01.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-19T20:53:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-19T20:53:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-19T20:53:01.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 4 closest matches: 2026-02-19T20:53:01.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_21 2026-02-19T20:53:01.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_25 2026-02-19T20:53:01.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_27 2026-02-19T20:53:01.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_29 2026-02-19T20:53:01.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-19T20:53:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:53:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:53:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:01.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-19T20:53:01.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-19T20:53:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:53:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-19T20:53:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:53:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:53:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:53:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:53:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:53:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:53:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:53:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40898], [object_number=37, tag_tid=1, entry_tid=40897], [object_number=36, tag_tid=1, entry_tid=40896], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:53:01.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:53:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-19T20:53:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:53:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-19T20:53:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:53:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:01.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:53:01.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:53:01.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:01.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:53:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:53:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:53:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:53:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40898], [object_number=37, tag_tid=1, entry_tid=40897], [object_number=36, tag_tid=1, entry_tid=40896], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:53:01.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:01.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:53:01.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:53:01.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40898' '!=' 'object_number=38, tag_tid=1, entry_tid=40898' 2026-02-19T20:53:01.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:01.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:53:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_2 2026-02-19T20:53:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_2 ']' 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:03.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:03.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:03.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:03.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:03.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:03.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-19T20:53:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-19T20:53:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-19T20:53:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-19T20:53:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_27 2026-02-19T20:53:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-19T20:53:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:53:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:53:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:03.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:03.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:03.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-19T20:53:03.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-19T20:53:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:53:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-19T20:53:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:53:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:53:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:53:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:03.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:53:03.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:53:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:53:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:53:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:53:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:53:03.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-19T20:53:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:53:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-19T20:53:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T20:53:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-19T20:53:03.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-19T20:53:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-19T20:53:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:53:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:53:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:53:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40898], [object_number=39, tag_tid=1, entry_tid=40895]]], state=connected] 2026-02-19T20:53:03.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.status 2026-02-19T20:53:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:53:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:53:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2026-02-19T20:53:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:53:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:53:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2026-02-19T20:53:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_2': 2026-02-19T20:53:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:53:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:53:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:53:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a6e5ab5571 2026-02-19T20:53:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a6e5ab5571 2026-02-19T20:53:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:53:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:53:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:53:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:53:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:43:39 2026 2026-02-19T20:53:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:43:39 2026 2026-02-19T20:53:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:44:40 2026 2026-02-19T20:53:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:53:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a6e5ab5571 2026-02-19T20:53:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:53:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:53:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 5a37e552-7be0-442d-9731-666818380f30 2026-02-19T20:53:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:53:04.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:04.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_2 snap 2026-02-19T20:53:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T20:53:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:53:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.export 2026-02-19T20:53:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_2.export 2026-02-19T20:53:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_2.export 2026-02-19T20:53:04.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.export 2026-02-19T20:53:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_2.export 2026-02-19T20:53:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_2.export 2026-02-19T20:53:05.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_2.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_2.export 2026-02-19T20:53:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:53:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_3 2026-02-19T20:53:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_3 snap 2026-02-19T20:53:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:53:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2026-02-19T20:53:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2026-02-19T20:53:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:53:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:08.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2026-02-19T20:53:08.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:08.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:08.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:08.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:53:08.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:53:08.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:53:08.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2026-02-19T20:53:08.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:53:08.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:53:08.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:08.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:08.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:53:08.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:53:08.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:08.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:53:08.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:08.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:53:08.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:08.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2026-02-19T20:53:08.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:53:08.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:08.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:08.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:08.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:08.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:08.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:08.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:08.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:08.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:08.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:08.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:08.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:53:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:53:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:53:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:53:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:08.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:08.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:08.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:08.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2026-02-19T20:53:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-19T20:53:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:53:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:08.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:53:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:53:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3 2026-02-19T20:53:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_36 2026-02-19T20:53:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_3 2026-02-19T20:53:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:53:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:53:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:08.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:08.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:08.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:08.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_3 2026-02-19T20:53:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_31 2026-02-19T20:53:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_3 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:53:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:53:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:53:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_3 2026-02-19T20:53:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:53:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:53:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:53:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:53:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:53:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:53:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:53:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_3 2026-02-19T20:53:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:53:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_3 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:53:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:53:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:53:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_3 2026-02-19T20:53:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_3 ']' 2026-02-19T20:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-19T20:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-19T20:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:09.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:09.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-19T20:53:09.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:09.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-19T20:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:09.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:09.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:09.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:09.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_36 2026-02-19T20:53:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-19T20:53:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:53:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:53:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:09.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:09.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_31 2026-02-19T20:53:09.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-19T20:53:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:53:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:53:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:53:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:53:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:53:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:53:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:53:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:53:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:53:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:09.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:09.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:53:09.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:09.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:53:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:09.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:09.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-19T20:53:09.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:09.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:09.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:09.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-19T20:53:09.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:09.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:09.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:09.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:09.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-19T20:53:09.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-19T20:53:09.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:09.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:09.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-19T20:53:09.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:53:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:53:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26990], [object_number=25, tag_tid=1, entry_tid=26989], [object_number=24, tag_tid=1, entry_tid=26988], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:53:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-19T20:53:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-19T20:53:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:09.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:09.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:09.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:09.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-19T20:53:09.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-19T20:53:09.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-19T20:53:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:53:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:53:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26990], [object_number=25, tag_tid=1, entry_tid=26989], [object_number=24, tag_tid=1, entry_tid=26988], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:09.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26990' '!=' '' 2026-02-19T20:53:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:53:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26990' = '' 2026-02-19T20:53:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:53:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:53:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:53:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:53:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:53:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:53:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:53:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:09.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:53:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:53:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26990 2026-02-19T20:53:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 26990 2026-02-19T20:53:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:53:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:53:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_3 2026-02-19T20:53:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_3 ']' 2026-02-19T20:53:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-19T20:53:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-19T20:53:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:09.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:09.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:09.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:09.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:09.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:09.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:09.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:09.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:09.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:09.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:09.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:09.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:09.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:09.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:09.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:09.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-19T20:53:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:09.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:09.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:09.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_36 2026-02-19T20:53:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-19T20:53:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:53:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:10.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_31 2026-02-19T20:53:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:10.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:10.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:53:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:53:10.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:10.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:10.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:10.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-19T20:53:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-19T20:53:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-19T20:53:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-19T20:53:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-19T20:53:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:10.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:53:10.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:53:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26990], [object_number=25, tag_tid=1, entry_tid=26989], [object_number=24, tag_tid=1, entry_tid=26988], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:10.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:10.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:53:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-19T20:53:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-19T20:53:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-19T20:53:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-19T20:53:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-19T20:53:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:53:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:53:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26990], [object_number=25, tag_tid=1, entry_tid=26989], [object_number=24, tag_tid=1, entry_tid=26988], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:10.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:10.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26990' '!=' 'object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:53:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_3 2026-02-19T20:53:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_3 ']' 2026-02-19T20:53:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-19T20:53:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-19T20:53:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:10.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:10.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-19T20:53:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-19T20:53:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:10.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:10.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:10.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:10.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_36 2026-02-19T20:53:10.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-19T20:53:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:53:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:10.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_31 2026-02-19T20:53:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:10.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:10.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-19T20:53:10.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:53:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:10.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:53:10.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:10.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:10.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:10.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-19T20:53:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-19T20:53:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:10.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-19T20:53:10.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-19T20:53:10.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:10.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:10.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-19T20:53:10.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:53:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:53:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26990], [object_number=25, tag_tid=1, entry_tid=26989], [object_number=24, tag_tid=1, entry_tid=26988], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:53:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-19T20:53:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-19T20:53:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:10.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-19T20:53:10.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-19T20:53:10.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:10.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-19T20:53:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:11.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:53:11.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:53:11.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:11.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:11.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26990], [object_number=25, tag_tid=1, entry_tid=26989], [object_number=24, tag_tid=1, entry_tid=26988], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:11.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:11.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:11.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26990' '!=' 'object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:11.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:11.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:53:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_3 2026-02-19T20:53:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_3 ']' 2026-02-19T20:53:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-19T20:53:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-19T20:53:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:11.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:11.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-19T20:53:11.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:11.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:11.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:11.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:11.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:11.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:11.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-19T20:53:11.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:11.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:11.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:11.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-19T20:53:11.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:11.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:11.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:11.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:11.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:11.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:11.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:11.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_36 2026-02-19T20:53:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-19T20:53:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:11.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:11.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:11.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:11.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:12.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:12.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_31 2026-02-19T20:53:12.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:12.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:12.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:53:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:53:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:12.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-19T20:53:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-19T20:53:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-19T20:53:12.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-19T20:53:12.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:12.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:12.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-19T20:53:12.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:53:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:53:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26990], [object_number=25, tag_tid=1, entry_tid=26989], [object_number=24, tag_tid=1, entry_tid=26988], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:12.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:53:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-19T20:53:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-19T20:53:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-19T20:53:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-19T20:53:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-19T20:53:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:53:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:53:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26990], [object_number=25, tag_tid=1, entry_tid=26989], [object_number=24, tag_tid=1, entry_tid=26988], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:12.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:12.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26990' '!=' 'object_number=26, tag_tid=1, entry_tid=26990' 2026-02-19T20:53:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:53:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_3 2026-02-19T20:53:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_3 ']' 2026-02-19T20:53:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-19T20:53:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-19T20:53:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-19T20:53:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-19T20:53:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:13.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:13.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-19T20:53:13.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:13.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:13.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:13.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:13.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:13.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:13.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:13.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:13.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:13.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:13.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_36 2026-02-19T20:53:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:13.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:13.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-19T20:53:13.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:53:13.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:13.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:13.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:13.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush mirror/image_31 2026-02-19T20:53:13.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:53:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-19T20:53:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-19T20:53:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-19T20:53:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-19T20:53:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:53:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:53:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:53:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-19T20:53:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-19T20:53:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T20:53:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-19T20:53:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-19T20:53:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:14.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-19T20:53:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:53:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:53:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26990], [object_number=27, tag_tid=1, entry_tid=26987]]], state=connected] 2026-02-19T20:53:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.status 2026-02-19T20:53:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:53:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:53:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2026-02-19T20:53:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:53:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:15.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2026-02-19T20:53:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_3': 2026-02-19T20:53:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:53:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:53:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:53:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 117c4f731110 2026-02-19T20:53:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.117c4f731110 2026-02-19T20:53:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:43:50 2026 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:43:50 2026 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:43:50 2026 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 117c4f731110 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:53:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 87bf0230-6085-45f3-89e1-3d5d3aeefa28 2026-02-19T20:53:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:53:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_3 snap 2026-02-19T20:53:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T20:53:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:53:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.export 2026-02-19T20:53:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_3.export 2026-02-19T20:53:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_3.export 2026-02-19T20:53:15.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.export 2026-02-19T20:53:15.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_3.export 2026-02-19T20:53:15.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:15.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_3.export 2026-02-19T20:53:15.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_3.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_3.export 2026-02-19T20:53:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:53:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_4 2026-02-19T20:53:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_4 snap 2026-02-19T20:53:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:53:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:53:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2026-02-19T20:53:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:53:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2026-02-19T20:53:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:53:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:53:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:20.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2026-02-19T20:53:20.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:20.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:20.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:53:20.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:53:20.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:53:20.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:53:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2026-02-19T20:53:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:53:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:53:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:20.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:53:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:53:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:53:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:53:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2026-02-19T20:53:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:53:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:53:20.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:53:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:53:20.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:20.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:20.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2026-02-19T20:53:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_4'] 2026-02-19T20:53:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:53:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4 2026-02-19T20:53:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:53:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:53:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:53:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:20.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:20.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:20.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:20.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4 2026-02-19T20:53:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 2 closest matches: 2026-02-19T20:53:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_45 2026-02-19T20:53:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_49 2026-02-19T20:53:20.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_4 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:53:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:53:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_4 2026-02-19T20:53:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 5 closest matches: 2026-02-19T20:53:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_40 2026-02-19T20:53:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_41 2026-02-19T20:53:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_42 2026-02-19T20:53:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_44 2026-02-19T20:53:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_48 2026-02-19T20:53:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_4 2026-02-19T20:53:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:53:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:53:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:53:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:53:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:53:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:53:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:53:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_4 2026-02-19T20:53:21.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:21.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_46 2026-02-19T20:53:21.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:21.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 1 2026-02-19T20:53:22.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:22.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:53:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2026-02-19T20:53:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:53:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:53:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:53:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2026-02-19T20:53:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_4'] 2026-02-19T20:53:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:53:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:22.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:22.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:53:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:53:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4 2026-02-19T20:53:22.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 2 closest matches: 2026-02-19T20:53:22.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_45 2026-02-19T20:53:22.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_49 2026-02-19T20:53:22.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_4 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:53:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:53:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_4 2026-02-19T20:53:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 5 closest matches: 2026-02-19T20:53:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_40 2026-02-19T20:53:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_41 2026-02-19T20:53:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_42 2026-02-19T20:53:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_44 2026-02-19T20:53:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_48 2026-02-19T20:53:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_4 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:22.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:22.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:22.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:22.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:22.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:53:22.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:53:22.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:53:22.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_4 2026-02-19T20:53:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_46 2026-02-19T20:53:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:22.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:22.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 2 2026-02-19T20:53:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:53:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2026-02-19T20:53:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:53:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:53:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:53:24.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:53:24.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:24.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2026-02-19T20:53:24.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_4'] 2026-02-19T20:53:24.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:53:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4 2026-02-19T20:53:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:53:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:53:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:24.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:24.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:24.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4 2026-02-19T20:53:24.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 2 closest matches: 2026-02-19T20:53:24.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_45 2026-02-19T20:53:24.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_49 2026-02-19T20:53:24.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:24.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:24.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_4 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:24.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:24.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:24.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:24.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:24.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:53:24.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:53:24.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_4 2026-02-19T20:53:24.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 5 closest matches: 2026-02-19T20:53:24.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_40 2026-02-19T20:53:24.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_41 2026-02-19T20:53:24.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_42 2026-02-19T20:53:24.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_44 2026-02-19T20:53:24.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_48 2026-02-19T20:53:24.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:24.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_4 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:53:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:53:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:53:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:53:24.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:53:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:53:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_4 2026-02-19T20:53:24.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:no valid command found; 1 closest matches: 2026-02-19T20:53:24.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status mirror/image_46 2026-02-19T20:53:24.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:24.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 4 2026-02-19T20:53:25.762 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.37668390369928095) > weight (0.25) 2026-02-19T20:53:25.763 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7319883559739643) > weight (0.25) 2026-02-19T20:53:25.763 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6836619286192408) > weight (0.25) 2026-02-19T20:53:25.763 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-19T20:53:25.763 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-19T20:53:25.763 INFO:tasks.rbd_mirror_thrash:waiting for 2.0197266654215906 secs before reviving daemons 2026-02-19T20:53:25.764 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T20:53:25.753+0000 7f282b3bf640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 83358) UID: 1000 2026-02-19T20:53:27.784 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-19T20:53:27.784 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:53:27.785 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-19T20:53:27.785 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-19T20:53:27.785 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-19T20:53:27.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-19T20:53:27.787 INFO:tasks.rbd_mirror_thrash:waiting for 107.59562735380165 secs before thrashing 2026-02-19T20:53:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:53:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2026-02-19T20:53:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:53:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:53:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:28.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:28.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:28.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:28.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2026-02-19T20:53:28.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:53:28.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:53:28.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:53:28.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:53:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:53:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:53:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:53:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_4 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_4 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:53:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_4 2026-02-19T20:53:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_4 ']' 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_4' 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2026-02-19T20:53:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:29.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_4 2026-02-19T20:53:29.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:29.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:29.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_4 2026-02-19T20:53:29.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_4 '' 2026-02-19T20:53:29.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:29.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:29.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_4 2026-02-19T20:53:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-19T20:53:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-19T20:53:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.status 2026-02-19T20:53:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-19T20:53:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.status 2026-02-19T20:53:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:53:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:53:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34339], [object_number=34, tag_tid=1, entry_tid=34338]]], state=connected] 2026-02-19T20:53:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34339], [object_number=34, tag_tid=1, entry_tid=34338]]], state=connected] 2026-02-19T20:53:29.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.status 2026-02-19T20:53:29.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:53:29.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2026-02-19T20:53:29.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:29.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_4 2026-02-19T20:53:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2026-02-19T20:53:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_4 2026-02-19T20:53:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-19T20:53:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-19T20:53:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:29.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.status 2026-02-19T20:53:29.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-19T20:53:29.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.status 2026-02-19T20:53:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:53:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:53:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34339], [object_number=34, tag_tid=1, entry_tid=34338]]], state=connected] 2026-02-19T20:53:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34339], [object_number=34, tag_tid=1, entry_tid=34338]]], state=connected] 2026-02-19T20:53:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.status 2026-02-19T20:53:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:53:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:53:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2026-02-19T20:53:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:53:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:53:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2026-02-19T20:53:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_4': 2026-02-19T20:53:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:53:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:53:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11a21838dfa9 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11a21838dfa9 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:43:57 2026 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:43:57 2026 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:44:58 2026 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:53:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11a21838dfa9 2026-02-19T20:53:30.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:53:30.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:53:30.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 29d990f3-8f04-4f5a-8770-52f9b8caef91 2026-02-19T20:53:30.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:53:30.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:30.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_4 snap 2026-02-19T20:53:30.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:30.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T20:53:30.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:30.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:53:30.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.export 2026-02-19T20:53:30.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_4.export 2026-02-19T20:53:30.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_4.export 2026-02-19T20:53:30.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.export 2026-02-19T20:53:30.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_4.export 2026-02-19T20:53:30.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_4.export 2026-02-19T20:53:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_4.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_4.export 2026-02-19T20:53:30.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:53:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_5 2026-02-19T20:53:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_5 snap 2026-02-19T20:53:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:53:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:53:30.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2026-02-19T20:53:30.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:30.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:53:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2026-02-19T20:53:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:53:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:53:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:31.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2026-02-19T20:53:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:53:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:53:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:53:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:53:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2026-02-19T20:53:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:53:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:53:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:53:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:53:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:53:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:53:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2026-02-19T20:53:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:53:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:53:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:53:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:53:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2026-02-19T20:53:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-19T20:53:31.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:53:31.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_5 2026-02-19T20:53:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:53:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:53:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_5 2026-02-19T20:53:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-19T20:53:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:53:31.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_5 2026-02-19T20:53:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:53:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:53:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:53:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_5 2026-02-19T20:53:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:53:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:53:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:53:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:53:32.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:53:32.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:53:32.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:53:32.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:32.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_5 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_5 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:53:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:53:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:53:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:53:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:53:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:53:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:53:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_5 2026-02-19T20:53:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:53:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_5 ']' 2026-02-19T20:53:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-19T20:53:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-19T20:53:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:32.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:32.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:32.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:32.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:32.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:32.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:32.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:32.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:32.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:32.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-19T20:53:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-19T20:53:32.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:32.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-19T20:53:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:32.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:32.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:32.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:32.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-19T20:53:32.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-19T20:53:32.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:32.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-19T20:53:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:53:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:53:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:32.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:32.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:32.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:32.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:32.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-19T20:53:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:32.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-19T20:53:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:32.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:32.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-19T20:53:32.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:32.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:32.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:32.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17874], [object_number=19, tag_tid=1, entry_tid=17871]]], state=connected] 2026-02-19T20:53:32.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17874], [object_number=17, tag_tid=1, entry_tid=17873], [object_number=16, tag_tid=1, entry_tid=17872], [object_number=19, tag_tid=1, entry_tid=17871]]], state=connected] 2026-02-19T20:53:32.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-19T20:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T20:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-19T20:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T20:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-19T20:53:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-19T20:53:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-19T20:53:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:32.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:32.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:32.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17874], [object_number=19, tag_tid=1, entry_tid=17871]]], state=connected] 2026-02-19T20:53:32.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17874], [object_number=17, tag_tid=1, entry_tid=17873], [object_number=16, tag_tid=1, entry_tid=17872], [object_number=19, tag_tid=1, entry_tid=17871]]], state=connected] 2026-02-19T20:53:32.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17874' 2026-02-19T20:53:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17874' 2026-02-19T20:53:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17874' '!=' '' 2026-02-19T20:53:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:53:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17874' = '' 2026-02-19T20:53:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=18, tag_tid=1, entry_tid=17874' 2026-02-19T20:53:32.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:53:32.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:53:32.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:32.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:53:32.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=18, tag_tid=1, entry_tid=17874' 2026-02-19T20:53:32.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:53:32.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:32.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:53:32.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:53:32.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:53:32.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:53:32.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=18, tag_tid=1, entry_tid=17874' 2026-02-19T20:53:32.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:53:32.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=17874 2026-02-19T20:53:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 1 -a 1 -le 17874 2026-02-19T20:53:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:53:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:53:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_5 2026-02-19T20:53:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:53:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_5 ']' 2026-02-19T20:53:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-19T20:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-19T20:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:32.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:32.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-19T20:53:32.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:32.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:32.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:32.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:32.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:32.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:32.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:32.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:32.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:32.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-19T20:53:32.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-19T20:53:32.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:32.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:32.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-19T20:53:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-19T20:53:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:53:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:53:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:53:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:53:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:53:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:53:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:53:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-19T20:53:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-19T20:53:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T20:53:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-19T20:53:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T20:53:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-19T20:53:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-19T20:53:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:33.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-19T20:53:33.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17874], [object_number=19, tag_tid=1, entry_tid=17871]]], state=connected] 2026-02-19T20:53:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17874], [object_number=19, tag_tid=1, entry_tid=17871]]], state=connected] 2026-02-19T20:53:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:53:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-19T20:53:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T20:53:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-19T20:53:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T20:53:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-19T20:53:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-19T20:53:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-19T20:53:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:33.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:33.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:33.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17874], [object_number=19, tag_tid=1, entry_tid=17871]]], state=connected] 2026-02-19T20:53:33.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17874], [object_number=19, tag_tid=1, entry_tid=17871]]], state=connected] 2026-02-19T20:53:33.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.status 2026-02-19T20:53:33.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:53:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:53:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2026-02-19T20:53:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:53:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:53:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2026-02-19T20:53:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_5': 2026-02-19T20:53:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:53:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:53:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:53:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a6c84e4df0 2026-02-19T20:53:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a6c84e4df0 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:44:06 2026 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:44:06 2026 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:49:46 2026 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a6c84e4df0 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:53:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: a7464a45-2b46-457c-98d6-156768f84501 2026-02-19T20:53:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:53:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_5 snap 2026-02-19T20:53:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T20:53:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:53:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.export 2026-02-19T20:53:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_5.export 2026-02-19T20:53:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_5.export 2026-02-19T20:53:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.export 2026-02-19T20:53:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_5.export 2026-02-19T20:53:34.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:34.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_5.export 2026-02-19T20:53:34.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_5.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_5.export 2026-02-19T20:53:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:53:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_6 2026-02-19T20:53:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_6 snap 2026-02-19T20:53:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:53:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2026-02-19T20:53:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2026-02-19T20:53:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:53:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:37.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2026-02-19T20:53:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:53:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:53:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:53:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2026-02-19T20:53:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:53:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:53:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:53:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:53:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:37.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2026-02-19T20:53:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-19T20:53:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:53:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6 2026-02-19T20:53:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:53:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:53:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:53:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6 2026-02-19T20:53:38.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:53:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:53:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:53:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:53:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:53:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:53:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:53:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_6 2026-02-19T20:53:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:53:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_6 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:53:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:53:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:53:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:53:38.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_6 2026-02-19T20:53:38.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:38.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:38.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_6 ']' 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:38.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:38.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:38.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:38.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-19T20:53:38.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:38.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:38.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:38.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:38.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:38.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:38.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-19T20:53:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-19T20:53:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:38.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-19T20:53:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-19T20:53:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-19T20:53:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-19T20:53:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:38.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:38.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20697], [object_number=16, tag_tid=1, entry_tid=20696], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:38.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:53:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-19T20:53:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-19T20:53:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-19T20:53:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-19T20:53:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-19T20:53:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20697], [object_number=16, tag_tid=1, entry_tid=20696], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:38.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:38.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20697' '!=' '' 2026-02-19T20:53:38.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:53:38.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20697' = '' 2026-02-19T20:53:38.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:38.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:53:38.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:53:38.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:38.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:53:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:53:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:38.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:53:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:53:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:53:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:53:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:53:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=20697 2026-02-19T20:53:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 20697 2026-02-19T20:53:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:53:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:53:38.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_6 2026-02-19T20:53:38.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:38.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:38.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_6 ']' 2026-02-19T20:53:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-19T20:53:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-19T20:53:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-19T20:53:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-19T20:53:38.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-19T20:53:38.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:38.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:38.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:38.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-19T20:53:38.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:38.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:38.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:38.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:38.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-19T20:53:38.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-19T20:53:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:38.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-19T20:53:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-19T20:53:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-19T20:53:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-19T20:53:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20697], [object_number=16, tag_tid=1, entry_tid=20696], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:53:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-19T20:53:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-19T20:53:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-19T20:53:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-19T20:53:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-19T20:53:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:38.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:38.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:38.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:38.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20697], [object_number=16, tag_tid=1, entry_tid=20696], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20697' '!=' 'object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:38.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:53:39.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_6 2026-02-19T20:53:39.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_6 ']' 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:39.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:39.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:39.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:39.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:39.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:39.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:39.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-19T20:53:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-19T20:53:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-19T20:53:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-19T20:53:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-19T20:53:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-19T20:53:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-19T20:53:39.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-19T20:53:39.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:39.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:39.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-19T20:53:39.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20697], [object_number=16, tag_tid=1, entry_tid=20696], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:39.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:39.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:53:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-19T20:53:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-19T20:53:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-19T20:53:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-19T20:53:39.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:39.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-19T20:53:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20697], [object_number=16, tag_tid=1, entry_tid=20696], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20697' '!=' 'object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:53:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_6 2026-02-19T20:53:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_6 ']' 2026-02-19T20:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-19T20:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-19T20:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:40.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:40.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:40.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-19T20:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:40.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:40.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:40.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-19T20:53:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-19T20:53:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:40.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:40.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-19T20:53:40.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-19T20:53:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-19T20:53:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-19T20:53:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-19T20:53:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-19T20:53:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-19T20:53:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20697], [object_number=16, tag_tid=1, entry_tid=20696], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:53:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-19T20:53:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-19T20:53:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-19T20:53:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-19T20:53:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-19T20:53:40.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:40.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:40.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:40.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:40.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:40.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=20697], [object_number=16, tag_tid=1, entry_tid=20696], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:40.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:40.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:40.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=20697' '!=' 'object_number=17, tag_tid=1, entry_tid=20697' 2026-02-19T20:53:40.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:40.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:53:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_6 2026-02-19T20:53:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_6 ']' 2026-02-19T20:53:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-19T20:53:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-19T20:53:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:42.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:42.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:42.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:42.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:42.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:42.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:42.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-19T20:53:42.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-19T20:53:42.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:42.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-19T20:53:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:42.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:42.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-19T20:53:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-19T20:53:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-19T20:53:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-19T20:53:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-19T20:53:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-19T20:53:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:42.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:42.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:42.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:42.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:42.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:53:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-19T20:53:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-19T20:53:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T20:53:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-19T20:53:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-19T20:53:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-19T20:53:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:53:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:53:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:42.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=20695], [object_number=18, tag_tid=1, entry_tid=20694]]], state=connected] 2026-02-19T20:53:42.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.status 2026-02-19T20:53:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:53:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:53:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2026-02-19T20:53:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:53:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:43.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2026-02-19T20:53:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_6': 2026-02-19T20:53:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:53:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11167c987454 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11167c987454 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:44:11 2026 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:44:11 2026 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:44:11 2026 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11167c987454 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: cff6005f-6f06-4520-8a2d-79f128f1e2a8 2026-02-19T20:53:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:53:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_6 snap 2026-02-19T20:53:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T20:53:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:53:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.export 2026-02-19T20:53:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_6.export 2026-02-19T20:53:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_6.export 2026-02-19T20:53:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.export 2026-02-19T20:53:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_6.export 2026-02-19T20:53:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_6.export 2026-02-19T20:53:43.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_6.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_6.export 2026-02-19T20:53:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:53:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_7 2026-02-19T20:53:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_7 snap 2026-02-19T20:53:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:53:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2026-02-19T20:53:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2026-02-19T20:53:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:53:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2026-02-19T20:53:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:53:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:53:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:53:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2026-02-19T20:53:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:53:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:53:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:53:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:53:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:53:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:53:48.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:48.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:53:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2026-02-19T20:53:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-19T20:53:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:53:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_7 2026-02-19T20:53:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:53:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:53:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:53:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:53:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:48.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:48.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:48.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:48.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_7 2026-02-19T20:53:49.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:53:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:53:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:53:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:53:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:53:49.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:53:49.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:53:49.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:49.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_7 2026-02-19T20:53:49.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_7 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:53:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:53:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:53:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:53:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_7 2026-02-19T20:53:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_7 ']' 2026-02-19T20:53:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-19T20:53:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-19T20:53:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:49.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-19T20:53:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:49.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:49.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:49.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:49.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-19T20:53:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-19T20:53:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-19T20:53:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-19T20:53:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-19T20:53:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:49.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-19T20:53:49.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-19T20:53:49.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:49.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:49.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-19T20:53:49.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:49.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:53:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:53:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=28, tag_tid=1, entry_tid=31852], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850], [object_number=29, tag_tid=1, entry_tid=31849]]], state=connected] 2026-02-19T20:53:49.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:49.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:53:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-19T20:53:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-19T20:53:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-19T20:53:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-19T20:53:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-19T20:53:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:53:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:53:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=28, tag_tid=1, entry_tid=31852], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850], [object_number=29, tag_tid=1, entry_tid=31849]]], state=connected] 2026-02-19T20:53:49.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=28, tag_tid=1, entry_tid=31852' '!=' '' 2026-02-19T20:53:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:53:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=28, tag_tid=1, entry_tid=31852' = '' 2026-02-19T20:53:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:53:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:53:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:49.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:53:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:53:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:53:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:53:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:53:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:53:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:53:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:53:49.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31852 2026-02-19T20:53:49.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 31852 2026-02-19T20:53:49.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:53:49.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:49.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_7 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_7 ']' 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-19T20:53:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-19T20:53:49.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-19T20:53:49.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:49.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-19T20:53:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:49.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-19T20:53:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-19T20:53:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-19T20:53:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-19T20:53:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-19T20:53:50.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:50.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-19T20:53:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:53:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:53:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=28, tag_tid=1, entry_tid=31852], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850], [object_number=29, tag_tid=1, entry_tid=31849]]], state=connected] 2026-02-19T20:53:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-19T20:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-19T20:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-19T20:53:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-19T20:53:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-19T20:53:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:53:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:53:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=28, tag_tid=1, entry_tid=31852], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850], [object_number=29, tag_tid=1, entry_tid=31849]]], state=connected] 2026-02-19T20:53:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=28, tag_tid=1, entry_tid=31852' '!=' 'object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:53:50.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_7 2026-02-19T20:53:50.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:50.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_7 ']' 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-19T20:53:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-19T20:53:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:50.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:50.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:50.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:50.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:50.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:50.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-19T20:53:50.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:50.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-19T20:53:50.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-19T20:53:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:53:50.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:53:50.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:50.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:50.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=28, tag_tid=1, entry_tid=31852], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850], [object_number=29, tag_tid=1, entry_tid=31849]]], state=connected] 2026-02-19T20:53:50.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:53:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-19T20:53:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-19T20:53:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:50.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-19T20:53:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-19T20:53:50.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:50.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-19T20:53:50.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:53:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:53:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=28, tag_tid=1, entry_tid=31852], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850], [object_number=29, tag_tid=1, entry_tid=31849]]], state=connected] 2026-02-19T20:53:50.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:50.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:50.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:50.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=28, tag_tid=1, entry_tid=31852' '!=' 'object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:50.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:50.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:53:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_7 2026-02-19T20:53:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_7 ']' 2026-02-19T20:53:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-19T20:53:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-19T20:53:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:51.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:51.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-19T20:53:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-19T20:53:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-19T20:53:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:51.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:51.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:51.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:51.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-19T20:53:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-19T20:53:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-19T20:53:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-19T20:53:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-19T20:53:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-19T20:53:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:51.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:53:51.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:53:51.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:51.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:51.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=28, tag_tid=1, entry_tid=31852], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850], [object_number=29, tag_tid=1, entry_tid=31849]]], state=connected] 2026-02-19T20:53:51.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:53:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-19T20:53:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-19T20:53:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-19T20:53:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-19T20:53:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-19T20:53:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:51.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:53:51.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:53:51.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:51.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:51.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=28, tag_tid=1, entry_tid=31852], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850], [object_number=29, tag_tid=1, entry_tid=31849]]], state=connected] 2026-02-19T20:53:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=28, tag_tid=1, entry_tid=31852' '!=' 'object_number=28, tag_tid=1, entry_tid=31852' 2026-02-19T20:53:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:53:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:53:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_7 2026-02-19T20:53:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_7 ']' 2026-02-19T20:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-19T20:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-19T20:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:53:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:53:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-19T20:53:53.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-19T20:53:53.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:53:53.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-19T20:53:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:53:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:53:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:53:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:53:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-19T20:53:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-19T20:53:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-19T20:53:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:53:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-19T20:53:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-19T20:53:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-19T20:53:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:53:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:53:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:53.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:53:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-19T20:53:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-19T20:53:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:53:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:53:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T20:53:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:53:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-19T20:53:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-19T20:53:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:53:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-19T20:53:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:53.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:53:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:53:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:53:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31851], [object_number=30, tag_tid=1, entry_tid=31850]]], state=connected] 2026-02-19T20:53:53.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.status 2026-02-19T20:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2026-02-19T20:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:53:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:54.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2026-02-19T20:53:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_7': 2026-02-19T20:53:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:53:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:53:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 117cc4dbfb2 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.117cc4dbfb2 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:44:18 2026 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:44:18 2026 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:45:19 2026 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:53:54.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 117cc4dbfb2 2026-02-19T20:53:54.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:53:54.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:53:54.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: fcf21ef8-faaa-42cc-91da-1f45eafbe464 2026-02-19T20:53:54.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:53:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_7 snap 2026-02-19T20:53:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T20:53:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:53:54.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.export 2026-02-19T20:53:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_7.export 2026-02-19T20:53:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_7.export 2026-02-19T20:53:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.export 2026-02-19T20:53:54.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:54.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_7.export 2026-02-19T20:53:55.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:53:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_7.export 2026-02-19T20:53:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_7.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_7.export 2026-02-19T20:53:55.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:53:55.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:53:55.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_8 2026-02-19T20:53:55.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_8 snap 2026-02-19T20:53:55.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:53:55.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:55.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:53:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:53:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:53:58.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2026-02-19T20:53:58.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:58.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:58.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:53:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2026-02-19T20:53:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:53:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:53:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:53:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:53:59.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2026-02-19T20:53:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:53:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:53:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:53:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:53:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:53:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:53:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2026-02-19T20:53:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:53:59.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:53:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:53:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:53:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:53:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:53:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:53:59.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:53:59.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:53:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:53:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:53:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:53:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:53:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:53:59.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:59.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:59.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:53:59.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2026-02-19T20:53:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:53:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:53:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:53:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:53:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_8 2026-02-19T20:53:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:53:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:53:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:53:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:53:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:53:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:53:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:53:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:53:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:53:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:53:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:53:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:53:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:53:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:53:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:53:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:53:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:53:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:53:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:53:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:53:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_8 2026-02-19T20:54:00.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:00.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:00.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_8 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:54:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:54:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_8 2026-02-19T20:54:00.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:54:00.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:54:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:54:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:54:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:54:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:54:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:54:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_8 2026-02-19T20:54:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:54:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:54:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_8 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:54:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:00.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:54:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_8 2026-02-19T20:54:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:54:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_8 ']' 2026-02-19T20:54:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-19T20:54:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-19T20:54:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:00.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:00.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:00.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-02-19T20:54:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:00.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:00.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:00.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:00.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:00.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:00.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:00.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:00.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:00.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:00.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:00.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:00.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:00.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:00.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_8 2026-02-19T20:54:00.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:00.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:00.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:00.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-19T20:54:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-19T20:54:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-19T20:54:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-19T20:54:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-19T20:54:00.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:54:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:54:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670], [object_number=29, tag_tid=1, entry_tid=32669], [object_number=28, tag_tid=1, entry_tid=32668]]], state=connected] 2026-02-19T20:54:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:54:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-19T20:54:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-19T20:54:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-19T20:54:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-19T20:54:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-19T20:54:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:54:00.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:54:00.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:00.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:00.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670], [object_number=29, tag_tid=1, entry_tid=32669], [object_number=28, tag_tid=1, entry_tid=32668]]], state=connected] 2026-02-19T20:54:00.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:00.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:00.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:00.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=32671' '!=' '' 2026-02-19T20:54:00.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:54:00.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=32671' = '' 2026-02-19T20:54:00.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:54:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:54:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:54:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:54:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:54:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:54:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:54:00.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:54:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:54:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=32671 2026-02-19T20:54:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 32671 2026-02-19T20:54:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:54:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:54:00.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_8 2026-02-19T20:54:00.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:00.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:00.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:54:00.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:00.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_8 ']' 2026-02-19T20:54:00.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-19T20:54:00.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-19T20:54:00.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:00.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:00.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:00.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:00.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:00.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:00.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:00.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:00.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:00.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:00.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-19T20:54:00.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:00.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-02-19T20:54:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:00.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:00.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:00.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:00.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:00.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:00.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_8 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:54:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-19T20:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-19T20:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-19T20:54:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-19T20:54:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-19T20:54:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:54:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:54:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670], [object_number=29, tag_tid=1, entry_tid=32669], [object_number=28, tag_tid=1, entry_tid=32668]]], state=connected] 2026-02-19T20:54:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:54:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-19T20:54:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-19T20:54:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-19T20:54:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-19T20:54:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-19T20:54:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:54:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:54:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670], [object_number=29, tag_tid=1, entry_tid=32669], [object_number=28, tag_tid=1, entry_tid=32668]]], state=connected] 2026-02-19T20:54:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:01.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:01.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=32671' '!=' 'object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:01.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:01.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:54:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_8 2026-02-19T20:54:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_8 ']' 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:01.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:01.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:01.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:01.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:01.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:01.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-02-19T20:54:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:01.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:01.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:01.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:01.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:01.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:01.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:01.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_8 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:01.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:01.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:01.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:01.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:01.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:01.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:01.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-19T20:54:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:01.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-19T20:54:01.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:54:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:54:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670], [object_number=29, tag_tid=1, entry_tid=32669], [object_number=28, tag_tid=1, entry_tid=32668]]], state=connected] 2026-02-19T20:54:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:54:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-19T20:54:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-19T20:54:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-19T20:54:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-19T20:54:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-19T20:54:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:54:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:54:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:01.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670], [object_number=29, tag_tid=1, entry_tid=32669], [object_number=28, tag_tid=1, entry_tid=32668]]], state=connected] 2026-02-19T20:54:01.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:01.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:01.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:01.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=32671' '!=' 'object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:01.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:01.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:54:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_8 2026-02-19T20:54:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:54:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_8 ']' 2026-02-19T20:54:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-19T20:54:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-19T20:54:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-19T20:54:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:02.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:02.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:02.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:02.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:02.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:02.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:02.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:02.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_8 2026-02-19T20:54:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:54:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:02.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:02.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:02.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-19T20:54:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-19T20:54:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-19T20:54:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-19T20:54:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-19T20:54:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:54:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:54:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670], [object_number=29, tag_tid=1, entry_tid=32669], [object_number=28, tag_tid=1, entry_tid=32668]]], state=connected] 2026-02-19T20:54:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:54:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-19T20:54:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-19T20:54:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-19T20:54:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-19T20:54:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-19T20:54:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:02.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:54:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:54:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670], [object_number=29, tag_tid=1, entry_tid=32669], [object_number=28, tag_tid=1, entry_tid=32668]]], state=connected] 2026-02-19T20:54:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=32671' '!=' 'object_number=31, tag_tid=1, entry_tid=32671' 2026-02-19T20:54:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:54:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_8 2026-02-19T20:54:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:54:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_8 ']' 2026-02-19T20:54:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-19T20:54:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-19T20:54:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:04.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:04.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-19T20:54:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:04.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:04.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:04.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:04.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-02-19T20:54:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-19T20:54:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_8 2026-02-19T20:54:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:04.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:54:04.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:04.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:04.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:54:04.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:04.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:04.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:04.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:04.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:04.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:04.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:04.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:04.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:04.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:04.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:04.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:04.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:04.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:04.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:04.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:04.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:04.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_8 2026-02-19T20:54:04.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:04.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-19T20:54:04.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:04.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:04.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-19T20:54:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-19T20:54:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-19T20:54:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-19T20:54:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:54:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:54:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:54:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-19T20:54:04.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:04.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:04.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:04.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-19T20:54:04.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:04.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:04.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T20:54:04.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:04.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-19T20:54:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-19T20:54:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-19T20:54:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:54:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:54:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:04.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32671], [object_number=30, tag_tid=1, entry_tid=32670]]], state=connected] 2026-02-19T20:54:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.status 2026-02-19T20:54:04.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:54:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:54:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2026-02-19T20:54:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:54:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:54:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:54:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:54:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:54:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2026-02-19T20:54:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_8': 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11a22aa2ba19 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11a22aa2ba19 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:44:29 2026 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:44:29 2026 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:45:30 2026 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11a22aa2ba19 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 8f2912f6-2d7d-4662-b7fb-048acb8eb9a6 2026-02-19T20:54:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:54:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_8 snap 2026-02-19T20:54:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T20:54:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:54:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:54:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.export 2026-02-19T20:54:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_8.export 2026-02-19T20:54:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_8.export 2026-02-19T20:54:05.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.export 2026-02-19T20:54:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:54:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_8.export 2026-02-19T20:54:06.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:54:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_8.export 2026-02-19T20:54:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_8.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_8.export 2026-02-19T20:54:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:54:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_9 2026-02-19T20:54:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_9 snap 2026-02-19T20:54:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:54:06.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:06.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:06.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:54:06.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:54:07.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2026-02-19T20:54:07.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:07.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2026-02-19T20:54:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:54:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:54:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:54:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2026-02-19T20:54:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:54:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:54:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:54:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2026-02-19T20:54:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:54:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:54:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:54:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:54:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:54:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:54:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2026-02-19T20:54:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:54:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:54:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:54:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:54:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:54:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2026-02-19T20:54:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9 2026-02-19T20:54:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:54:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:54:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:54:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:54:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:08.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:08.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:08.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:08.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9 2026-02-19T20:54:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:09.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_9 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:54:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:54:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:54:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:54:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_9 2026-02-19T20:54:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:54:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:54:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:54:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:54:09.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_9 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_9 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:54:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:54:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:54:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:54:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_9 2026-02-19T20:54:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_9 ']' 2026-02-19T20:54:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-19T20:54:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-19T20:54:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-19T20:54:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:09.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:09.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:09.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:09.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:09.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:09.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:09.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:09.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:09.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:09.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:09.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:09.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:09.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:09.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:09.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:09.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:09.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-19T20:54:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-19T20:54:09.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-19T20:54:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-19T20:54:09.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:09.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:09.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-19T20:54:09.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:54:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:54:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:09.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17800], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798], [object_number=17, tag_tid=1, entry_tid=17797]]], state=connected] 2026-02-19T20:54:09.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:54:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-19T20:54:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-19T20:54:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-19T20:54:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-19T20:54:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-19T20:54:09.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:54:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:54:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17800], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798], [object_number=17, tag_tid=1, entry_tid=17797]]], state=connected] 2026-02-19T20:54:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=17800' '!=' '' 2026-02-19T20:54:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:54:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=17800' = '' 2026-02-19T20:54:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:54:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:54:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:54:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:54:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:54:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:54:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:54:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:54:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:09.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:54:09.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:09.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=17800 2026-02-19T20:54:09.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 17800 2026-02-19T20:54:09.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:54:09.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:09.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:54:09.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_9 2026-02-19T20:54:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_9 ']' 2026-02-19T20:54:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-19T20:54:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-19T20:54:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:09.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:09.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:09.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:09.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:09.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:09.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:09.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-19T20:54:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:09.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:09.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:09.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:09.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-19T20:54:09.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:09.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:10.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:10.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:10.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-19T20:54:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-19T20:54:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-19T20:54:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-19T20:54:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:10.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-19T20:54:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:54:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:54:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17800], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798], [object_number=17, tag_tid=1, entry_tid=17797]]], state=connected] 2026-02-19T20:54:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:54:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-19T20:54:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-19T20:54:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-19T20:54:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-19T20:54:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-19T20:54:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:54:10.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:54:10.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:10.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:10.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17800], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798], [object_number=17, tag_tid=1, entry_tid=17797]]], state=connected] 2026-02-19T20:54:10.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=17800' '!=' 'object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:54:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_9 2026-02-19T20:54:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_9 ']' 2026-02-19T20:54:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-19T20:54:10.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-19T20:54:10.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:10.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:10.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:10.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:10.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:10.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-19T20:54:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:10.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:10.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:10.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-19T20:54:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:10.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:10.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:10.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:10.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:10.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-02-19T20:54:10.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:54:10.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:10.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:10.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:10.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:54:10.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:10.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:10.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:10.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:54:10.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:10.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:54:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:54:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-19T20:54:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-19T20:54:10.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:10.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:10.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:10.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:10.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-19T20:54:10.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-19T20:54:10.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:10.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-19T20:54:10.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:54:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:54:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17800], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798], [object_number=17, tag_tid=1, entry_tid=17797]]], state=connected] 2026-02-19T20:54:10.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:54:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-19T20:54:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:10.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-19T20:54:10.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:10.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:10.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:10.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:10.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-19T20:54:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-19T20:54:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-19T20:54:10.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:54:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:54:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17800], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798], [object_number=17, tag_tid=1, entry_tid=17797]]], state=connected] 2026-02-19T20:54:10.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:10.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:10.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:10.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=17800' '!=' 'object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:10.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:10.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:54:11.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_9 2026-02-19T20:54:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_9 ']' 2026-02-19T20:54:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-19T20:54:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-19T20:54:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:11.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:11.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:11.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:11.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:11.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-19T20:54:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:11.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:11.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:11.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:11.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:11.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-19T20:54:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:11.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:11.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-02-19T20:54:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:54:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:11.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:11.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:11.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:11.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:11.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:11.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-19T20:54:11.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:11.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-19T20:54:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-19T20:54:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-19T20:54:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:11.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:11.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-19T20:54:11.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:54:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:54:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17800], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798], [object_number=17, tag_tid=1, entry_tid=17797]]], state=connected] 2026-02-19T20:54:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:11.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:54:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-19T20:54:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-19T20:54:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-19T20:54:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-19T20:54:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:11.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:11.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-19T20:54:11.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:54:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:54:11.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:11.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:11.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17800], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798], [object_number=17, tag_tid=1, entry_tid=17797]]], state=connected] 2026-02-19T20:54:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:11.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:11.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=17800' '!=' 'object_number=16, tag_tid=1, entry_tid=17800' 2026-02-19T20:54:11.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:11.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:54:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_9 2026-02-19T20:54:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_9 ']' 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:13.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:13.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:13.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:13.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-19T20:54:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:13.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-19T20:54:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:13.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:13.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:13.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:13.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-19T20:54:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:13.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-02-19T20:54:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:54:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:54:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-02-19T20:54:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-19T20:54:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-19T20:54:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-19T20:54:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-19T20:54:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-19T20:54:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:13.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:54:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:54:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:54:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-19T20:54:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-19T20:54:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T20:54:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-19T20:54:13.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-19T20:54:13.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:13.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:13.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-19T20:54:13.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:13.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:54:13.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:54:13.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:13.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:13.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17799], [object_number=18, tag_tid=1, entry_tid=17798]]], state=connected] 2026-02-19T20:54:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.status 2026-02-19T20:54:13.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:54:13.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:54:13.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2026-02-19T20:54:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:54:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:54:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:54:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:54:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2026-02-19T20:54:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_9': 2026-02-19T20:54:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:54:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:54:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:54:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a6516cffab 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a6516cffab 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:44:38 2026 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:44:38 2026 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:44:38 2026 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a6516cffab 2026-02-19T20:54:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:54:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:54:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 4b58db02-9b2f-48f8-ac5f-70324fc638b5 2026-02-19T20:54:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:54:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_9 snap 2026-02-19T20:54:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T20:54:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:54:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:54:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.export 2026-02-19T20:54:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_9.export 2026-02-19T20:54:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_9.export 2026-02-19T20:54:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.export 2026-02-19T20:54:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:54:15.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_9.export 2026-02-19T20:54:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:54:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_9.export 2026-02-19T20:54:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_9.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_9.export 2026-02-19T20:54:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:54:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_10 2026-02-19T20:54:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_10 snap 2026-02-19T20:54:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:54:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:54:15.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:54:17.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2026-02-19T20:54:17.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:17.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:17.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:17.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2026-02-19T20:54:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:54:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:54:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:54:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:54:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:54:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2026-02-19T20:54:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:54:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:54:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2026-02-19T20:54:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-19T20:54:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2026-02-19T20:54:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:54:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:54:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:54:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:19.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:19.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:19.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2026-02-19T20:54:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:54:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:54:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:54:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:54:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_10 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_10 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:54:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:54:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:54:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:54:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:54:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:54:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_10 2026-02-19T20:54:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_10 ']' 2026-02-19T20:54:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-19T20:54:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-19T20:54:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-19T20:54:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:19.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:19.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:19.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-19T20:54:19.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-19T20:54:19.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:19.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-19T20:54:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:19.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:19.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:19.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:19.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:19.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-19T20:54:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-19T20:54:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-19T20:54:19.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:19.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:19.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:19.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:19.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-19T20:54:19.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-19T20:54:19.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-19T20:54:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:54:19.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:54:19.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:19.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:19.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44544], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542], [object_number=41, tag_tid=1, entry_tid=44541]]], state=connected] 2026-02-19T20:54:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:54:19.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-19T20:54:19.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:19.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:19.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:19.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-19T20:54:19.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:19.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:19.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:19.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:19.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-19T20:54:19.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-19T20:54:19.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:19.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-19T20:54:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:54:19.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:54:19.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:19.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:19.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44544], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542], [object_number=41, tag_tid=1, entry_tid=44541]]], state=connected] 2026-02-19T20:54:19.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:19.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=44544' '!=' '' 2026-02-19T20:54:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:54:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=44544' = '' 2026-02-19T20:54:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:54:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:54:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:19.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:54:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:54:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:54:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:54:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:54:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:54:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:54:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:54:19.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=44544 2026-02-19T20:54:19.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 44544 2026-02-19T20:54:19.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:54:19.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:19.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:54:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_10 2026-02-19T20:54:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_10 ']' 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-19T20:54:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-19T20:54:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-19T20:54:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:19.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:19.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-19T20:54:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-19T20:54:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-19T20:54:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-19T20:54:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-19T20:54:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-19T20:54:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:54:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:54:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44544], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542], [object_number=41, tag_tid=1, entry_tid=44541]]], state=connected] 2026-02-19T20:54:19.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-19T20:54:19.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-19T20:54:19.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:19.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-19T20:54:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:54:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:54:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44544], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542], [object_number=41, tag_tid=1, entry_tid=44541]]], state=connected] 2026-02-19T20:54:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=44544' '!=' 'object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:54:20.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_10 2026-02-19T20:54:20.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:20.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:20.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_10 ']' 2026-02-19T20:54:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-19T20:54:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-19T20:54:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-19T20:54:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-19T20:54:20.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-19T20:54:20.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:20.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:20.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:20.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-19T20:54:20.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:20.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:20.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-19T20:54:20.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-19T20:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-19T20:54:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-19T20:54:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-19T20:54:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:20.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-19T20:54:20.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:54:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:54:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44544], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542], [object_number=41, tag_tid=1, entry_tid=44541]]], state=connected] 2026-02-19T20:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:20.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:54:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-19T20:54:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-19T20:54:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-19T20:54:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-19T20:54:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-19T20:54:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:54:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:54:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44544], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542], [object_number=41, tag_tid=1, entry_tid=44541]]], state=connected] 2026-02-19T20:54:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=44544' '!=' 'object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:54:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_10 2026-02-19T20:54:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_10 ']' 2026-02-19T20:54:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-19T20:54:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-19T20:54:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-19T20:54:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-19T20:54:21.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-19T20:54:21.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:21.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-19T20:54:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:21.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-19T20:54:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-19T20:54:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-19T20:54:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-19T20:54:21.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-19T20:54:21.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:21.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-19T20:54:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:54:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:54:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44544], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542], [object_number=41, tag_tid=1, entry_tid=44541]]], state=connected] 2026-02-19T20:54:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:54:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-19T20:54:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-19T20:54:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-19T20:54:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-19T20:54:21.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:21.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:21.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-19T20:54:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:54:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:54:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:21.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=44544], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542], [object_number=41, tag_tid=1, entry_tid=44541]]], state=connected] 2026-02-19T20:54:21.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=44544' '!=' 'object_number=40, tag_tid=1, entry_tid=44544' 2026-02-19T20:54:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:54:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_10 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_10 ']' 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:54:23.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:23.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:23.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:23.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:23.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:23.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-19T20:54:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-19T20:54:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:54:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:23.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:54:23.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:23.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:23.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:23.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:23.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:54:23.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:54:23.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:23.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:23.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:23.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:23.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:23.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:23.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-19T20:54:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-19T20:54:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-19T20:54:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:54:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-19T20:54:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-19T20:54:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-19T20:54:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:54:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:54:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-19T20:54:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-19T20:54:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:54:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-19T20:54:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:54:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:54:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:54:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:23.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=44543], [object_number=42, tag_tid=1, entry_tid=44542]]], state=connected] 2026-02-19T20:54:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.status 2026-02-19T20:54:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:54:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:54:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2026-02-19T20:54:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:54:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:54:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:54:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:54:24.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_10': 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 111670481b91 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.111670481b91 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:44:42 2026 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:44:42 2026 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:45:43 2026 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 111670481b91 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:54:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:54:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 709b76a5-68e5-40f2-a913-0f5552f2a4cc 2026-02-19T20:54:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:54:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_10 snap 2026-02-19T20:54:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T20:54:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:54:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:54:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.export 2026-02-19T20:54:24.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_10.export 2026-02-19T20:54:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_10.export 2026-02-19T20:54:24.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.export 2026-02-19T20:54:24.812 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5967370139352156) > weight (0.25) 2026-02-19T20:54:24.812 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-19T20:54:24.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-19T20:54:24.812 INFO:tasks.rbd_mirror_thrash:waiting for 9.007783948770031 secs before reviving daemons 2026-02-19T20:54:24.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:54:24.801+0000 7efcc41cb640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 19426) UID: 1000 2026-02-19T20:54:24.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:54:24.801+0000 7efcbc9bc640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x560a8a6cee00 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-19T20:54:24.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:54:24.801+0000 7efcbc9bc640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x560a8a6cee00 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-19T20:54:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:54:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_10.export 2026-02-19T20:54:25.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:54:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_10.export 2026-02-19T20:54:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_10.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_10.export 2026-02-19T20:54:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:54:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:54:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_11 2026-02-19T20:54:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_11 snap 2026-02-19T20:54:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:54:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:54:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:54:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:54:28.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2026-02-19T20:54:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:54:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:54:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:54:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:54:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:54:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:54:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2026-02-19T20:54:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:54:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:54:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:54:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:54:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:54:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:54:28.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:54:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-19T20:54:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:54:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:54:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:54:28.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:28.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:28.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:28.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:28.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:28.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:28.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:28.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:54:28.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:54:28.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-19T20:54:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-19T20:54:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:54:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:28.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:28.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:28.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:28.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:28.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:28.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:54:28.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:54:28.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:28.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-19T20:54:28.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:54:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-19T20:54:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:54:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:54:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:54:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:28.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:28.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:28.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:28.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-19T20:54:29.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:29.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:29.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_11 2026-02-19T20:54:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:54:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:54:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:54:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:54:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:54:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:54:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:54:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_11 2026-02-19T20:54:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:29.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:29.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:29.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 1 2026-02-19T20:54:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:54:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-19T20:54:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:54:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:54:30.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-19T20:54:30.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:30.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:30.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-19T20:54:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:54:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:54:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:54:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:54:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-19T20:54:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:54:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-19T20:54:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:54:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:30.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:54:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-19T20:54:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_11 2026-02-19T20:54:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:54:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:54:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:54:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:54:30.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:30.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:30.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_11 2026-02-19T20:54:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 2 2026-02-19T20:54:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:54:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:32.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:32.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:32.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:32.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:32.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:32.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:54:32.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:54:32.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:32.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:32.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:32.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:32.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-19T20:54:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-19T20:54:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:54:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:54:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:54:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-19T20:54:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:54:32.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-19T20:54:32.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_11 2026-02-19T20:54:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:54:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:54:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:54:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:54:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:54:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:32.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:32.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_11 2026-02-19T20:54:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 4 2026-02-19T20:54:33.820 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-19T20:54:33.821 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:54:33.821 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-19T20:54:33.821 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-19T20:54:33.821 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-19T20:54:33.822 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-19T20:54:33.822 INFO:tasks.rbd_mirror_thrash:waiting for 75.57526414196174 secs before thrashing 2026-02-19T20:54:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:36.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:54:36.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:36.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-19T20:54:36.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:54:36.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:36.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:36.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:36.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:36.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:36.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:36.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:36.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:36.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:36.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:36.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:36.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:36.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:36.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:36.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:54:36.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:36.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:36.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:36.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:54:36.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:36.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:36.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:36.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:36.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:36.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:36.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:36.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:54:36.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:54:36.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:36.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-19T20:54:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:54:36.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:36.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:36.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:36.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:36.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-19T20:54:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:54:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:36.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:36.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:36.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-19T20:54:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_11 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:54:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:54:36.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:36.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:36.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:36.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_11 2026-02-19T20:54:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:36.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:36.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 8 2026-02-19T20:54:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:54:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-19T20:54:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:45.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:45.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-19T20:54:45.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:54:45.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:45.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:45.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:54:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:54:45.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-19T20:54:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:54:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:54:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:45.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:45.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:45.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-19T20:54:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_11 2026-02-19T20:54:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:54:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:54:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:54:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:54:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_11 2026-02-19T20:54:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 8 2026-02-19T20:54:53.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:54:53.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:54:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-19T20:54:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:54:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:54:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:54:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:54:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:54:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:54:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-19T20:54:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-19T20:54:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:54:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:54:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:54:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:54:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:54:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:54:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:54:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:54:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:54:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:54:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:54:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-19T20:54:53.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:53.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:53.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:54:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:54:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:54:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:54:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:54:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:54:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:54:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-19T20:54:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:53.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:54:53.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_11 2026-02-19T20:54:53.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:54:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:54:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:54:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:54:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:54:53.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:54:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_11 2026-02-19T20:54:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:54:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:54:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:54:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:54:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 8 2026-02-19T20:55:01.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:55:01.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:55:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:55:01.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:01.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:01.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:01.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:01.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-19T20:55:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:55:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:55:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-19T20:55:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:55:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:01.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:01.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:55:01.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:55:01.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:01.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:01.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:01.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:01.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-19T20:55:01.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:55:01.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:55:01.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:55:01.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:55:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:55:01.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:55:01.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_11 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_11 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:55:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:55:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:55:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:55:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:55:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:55:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_11 2026-02-19T20:55:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_11 ']' 2026-02-19T20:55:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-19T20:55:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-19T20:55:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-19T20:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:01.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:55:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-19T20:55:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-19T20:55:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-19T20:55:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-19T20:55:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-19T20:55:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:02.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:02.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:02.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:02.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:02.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:02.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:02.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=22534' '!=' '' 2026-02-19T20:55:02.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:55:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=22534' = '' 2026-02-19T20:55:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:55:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:55:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:55:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:55:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:55:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:55:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:55:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=22534 2026-02-19T20:55:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 22534 2026-02-19T20:55:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:55:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:55:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_11 2026-02-19T20:55:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_11 ']' 2026-02-19T20:55:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-19T20:55:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-19T20:55:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-19T20:55:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:02.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:55:02.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:02.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-19T20:55:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:02.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-19T20:55:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:02.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-19T20:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-19T20:55:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-19T20:55:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:02.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:02.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:02.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:02.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:02.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:02.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:02.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:02.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=22534' '!=' 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:02.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:02.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:55:02.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_11 2026-02-19T20:55:02.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:02.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:02.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:02.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:02.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_11 ']' 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-19T20:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:03.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:03.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:03.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:03.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:03.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:55:03.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:03.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-19T20:55:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:03.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-19T20:55:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-19T20:55:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-19T20:55:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-19T20:55:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:03.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:03.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:03.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=22534' '!=' 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:03.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:03.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:55:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_11 2026-02-19T20:55:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_11 ']' 2026-02-19T20:55:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-19T20:55:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-19T20:55:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-19T20:55:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:04.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:04.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:04.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:04.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:55:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-19T20:55:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:04.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-19T20:55:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-19T20:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-19T20:55:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-19T20:55:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=22534' '!=' 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:55:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_11 2026-02-19T20:55:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_11 ']' 2026-02-19T20:55:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-19T20:55:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-19T20:55:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:05.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:05.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-19T20:55:05.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:05.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:05.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:05.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:05.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:05.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:55:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-19T20:55:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-19T20:55:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:06.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:06.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:06.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-19T20:55:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-19T20:55:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:06.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:06.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:06.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=22534' '!=' 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:06.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:06.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:55:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_11 2026-02-19T20:55:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_11 ']' 2026-02-19T20:55:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-19T20:55:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-19T20:55:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-19T20:55:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:55:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-19T20:55:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:08.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:08.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:08.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:08.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-19T20:55:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-19T20:55:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:08.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-19T20:55:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:08.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:08.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:08.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:08.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:08.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:08.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=22534], [object_number=21, tag_tid=1, entry_tid=22533], [object_number=20, tag_tid=1, entry_tid=22532], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=22534' '!=' 'object_number=22, tag_tid=1, entry_tid=22534' 2026-02-19T20:55:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:55:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_11 2026-02-19T20:55:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_11 ']' 2026-02-19T20:55:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-19T20:55:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-19T20:55:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-19T20:55:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:10.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:10.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:10.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:10.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:10.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-19T20:55:10.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:10.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:10.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-19T20:55:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-19T20:55:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-19T20:55:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:10.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-19T20:55:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-19T20:55:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T20:55:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-19T20:55:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-19T20:55:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-19T20:55:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:10.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:55:10.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:55:10.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:10.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:10.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=22534], [object_number=23, tag_tid=1, entry_tid=22531]]], state=connected] 2026-02-19T20:55:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.status 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:55:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:55:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2026-02-19T20:55:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_11': 2026-02-19T20:55:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:55:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:55:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:55:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 117c4cd3705c 2026-02-19T20:55:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.117c4cd3705c 2026-02-19T20:55:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:55:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:55:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:55:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:55:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:44:56 2026 2026-02-19T20:55:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:44:56 2026 2026-02-19T20:55:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:44:56 2026 2026-02-19T20:55:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:55:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 117c4cd3705c 2026-02-19T20:55:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:55:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:55:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 9f8f4f28-edad-4695-a302-60801454ca0c 2026-02-19T20:55:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:55:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_11 snap 2026-02-19T20:55:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T20:55:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:55:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.export 2026-02-19T20:55:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_11.export 2026-02-19T20:55:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_11.export 2026-02-19T20:55:11.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.export 2026-02-19T20:55:11.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:55:11.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_11.export 2026-02-19T20:55:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:55:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_11.export 2026-02-19T20:55:12.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_11.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_11.export 2026-02-19T20:55:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:55:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_12 2026-02-19T20:55:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_12 snap 2026-02-19T20:55:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:55:15.384 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7284678774378243) > weight (0.25) 2026-02-19T20:55:15.384 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-19T20:55:15.385 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-19T20:55:15.385 INFO:tasks.rbd_mirror_thrash:waiting for 5.947886775718337 secs before reviving daemons 2026-02-19T20:55:15.385 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T20:55:15.372+0000 7fc1d05ee640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 84794) UID: 1000 2026-02-19T20:55:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2026-02-19T20:55:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2026-02-19T20:55:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:15.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:55:15.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:55:15.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:15.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:55:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2026-02-19T20:55:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:55:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:55:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:55:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2026-02-19T20:55:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:55:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:55:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:55:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:55:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:55:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:55:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:55:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:55:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:55:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-19T20:55:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:55:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2026-02-19T20:55:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:55:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:55:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:55:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:16.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:16.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:16.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2026-02-19T20:55:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:55:17.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_12 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:17.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:55:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:55:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_12 2026-02-19T20:55:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:55:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:55:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:55:17.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:55:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:55:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:55:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:55:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_12 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_12 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:55:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:55:17.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:17.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:55:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_12 2026-02-19T20:55:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_12 ']' 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:17.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:17.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:17.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-19T20:55:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:17.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:17.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:17.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:17.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:17.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:17.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-19T20:55:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-19T20:55:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-19T20:55:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:17.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:17.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:17.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:17.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:17.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=34754], [object_number=33, tag_tid=1, entry_tid=34753], [object_number=32, tag_tid=1, entry_tid=34752], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:17.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:17.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-19T20:55:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-19T20:55:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-19T20:55:17.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-19T20:55:17.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:17.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:17.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-19T20:55:17.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=34754], [object_number=33, tag_tid=1, entry_tid=34753], [object_number=32, tag_tid=1, entry_tid=34752], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:17.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=34754' '!=' '' 2026-02-19T20:55:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:55:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=34754' = '' 2026-02-19T20:55:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:55:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:17.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:55:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:55:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:55:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:55:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:55:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:55:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34754 2026-02-19T20:55:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 34754 2026-02-19T20:55:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:55:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:55:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_12 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_12 ']' 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:17.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:17.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:17.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:17.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:17.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:17.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:17.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:17.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-19T20:55:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-19T20:55:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:18.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-19T20:55:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-19T20:55:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-19T20:55:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:18.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=34754], [object_number=33, tag_tid=1, entry_tid=34753], [object_number=32, tag_tid=1, entry_tid=34752], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-19T20:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-19T20:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-19T20:55:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-19T20:55:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-19T20:55:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:18.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:18.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:18.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:18.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=34754], [object_number=33, tag_tid=1, entry_tid=34753], [object_number=32, tag_tid=1, entry_tid=34752], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=34754' '!=' 'object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:55:18.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_12 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_12 ']' 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:18.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-19T20:55:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:18.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-19T20:55:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-19T20:55:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-19T20:55:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-19T20:55:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=34754], [object_number=33, tag_tid=1, entry_tid=34753], [object_number=32, tag_tid=1, entry_tid=34752], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:18.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-19T20:55:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-19T20:55:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:18.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-19T20:55:18.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-19T20:55:18.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-19T20:55:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=34754], [object_number=33, tag_tid=1, entry_tid=34753], [object_number=32, tag_tid=1, entry_tid=34752], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=34754' '!=' 'object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:55:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_12 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_12 ']' 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:19.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:19.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:19.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:19.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:19.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:19.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:19.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-19T20:55:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-19T20:55:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-19T20:55:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-19T20:55:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:19.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-19T20:55:19.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:19.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:19.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:19.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:19.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:19.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=34754], [object_number=33, tag_tid=1, entry_tid=34753], [object_number=32, tag_tid=1, entry_tid=34752], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:19.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:19.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:19.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-19T20:55:19.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:19.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:19.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:19.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-19T20:55:19.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:19.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:19.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:19.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:19.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-19T20:55:19.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-19T20:55:19.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-19T20:55:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=34754], [object_number=33, tag_tid=1, entry_tid=34753], [object_number=32, tag_tid=1, entry_tid=34752], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=34754' '!=' 'object_number=34, tag_tid=1, entry_tid=34754' 2026-02-19T20:55:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:55:21.333 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-19T20:55:21.333 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:55:21.334 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-19T20:55:21.334 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-19T20:55:21.334 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-19T20:55:21.335 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-19T20:55:21.335 INFO:tasks.rbd_mirror_thrash:waiting for 86.88947580630924 secs before thrashing 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_12 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_12 ']' 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-19T20:55:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:21.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:21.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-19T20:55:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:21.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-19T20:55:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-19T20:55:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-19T20:55:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-19T20:55:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-19T20:55:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:21.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:21.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-19T20:55:21.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:21.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-19T20:55:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:21.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:21.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-19T20:55:21.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:21.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:21.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:21.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:21.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:21.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=34754], [object_number=35, tag_tid=1, entry_tid=34751]]], state=connected] 2026-02-19T20:55:21.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.status 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:55:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:55:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_12': 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11a26f19009e 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11a26f19009e 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:55:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:45:02 2026 2026-02-19T20:55:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:45:02 2026 2026-02-19T20:55:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:46:03 2026 2026-02-19T20:55:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:55:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11a26f19009e 2026-02-19T20:55:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:55:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:55:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: e3d71bea-907e-4390-b8ea-b164e5f8ebe0 2026-02-19T20:55:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:55:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_12 snap 2026-02-19T20:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T20:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.export 2026-02-19T20:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_12.export 2026-02-19T20:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_12.export 2026-02-19T20:55:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.export 2026-02-19T20:55:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:55:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_12.export 2026-02-19T20:55:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:55:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_12.export 2026-02-19T20:55:23.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_12.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_12.export 2026-02-19T20:55:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:55:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_13 2026-02-19T20:55:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_13 snap 2026-02-19T20:55:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:55:27.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:55:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2026-02-19T20:55:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:55:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:55:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:55:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2026-02-19T20:55:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:55:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:55:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:55:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:55:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:55:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:55:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-19T20:55:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:55:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:55:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:55:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-19T20:55:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-19T20:55:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:55:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-02-19T20:55:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:55:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:55:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:28.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:28.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:28.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:55:28.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:28.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:28.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:55:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:55:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:28.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-02-19T20:55:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:55:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:55:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:55:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:55:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:55:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:55:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:55:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_13 2026-02-19T20:55:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:55:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_13 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:55:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:55:29.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_13 2026-02-19T20:55:29.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:29.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:29.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:29.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:29.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_13 ']' 2026-02-19T20:55:29.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-19T20:55:29.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-19T20:55:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-19T20:55:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:29.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-19T20:55:29.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:29.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:29.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-19T20:55:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-19T20:55:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:29.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:29.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:29.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:29.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:29.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:29.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-19T20:55:29.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:29.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-19T20:55:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:29.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:29.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:29.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:29.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:29.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34813' '!=' '' 2026-02-19T20:55:29.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:55:29.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34813' = '' 2026-02-19T20:55:29.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:29.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:29.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:55:29.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:29.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:55:29.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:29.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:55:29.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:55:29.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:29.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:55:29.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:29.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:55:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:55:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=34813 2026-02-19T20:55:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 34813 2026-02-19T20:55:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:55:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:29.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:55:29.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_13 2026-02-19T20:55:29.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:29.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:29.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:29.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:29.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_13 ']' 2026-02-19T20:55:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-19T20:55:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-19T20:55:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-19T20:55:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:29.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:29.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:29.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:29.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:29.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:29.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:29.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:29.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:29.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:29.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:29.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:29.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:29.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:29.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:29.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:29.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:29.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:29.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-19T20:55:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:29.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:29.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:29.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:29.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-19T20:55:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-19T20:55:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:29.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-19T20:55:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-19T20:55:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:29.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:29.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:29.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:29.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:29.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:29.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:29.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:29.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34813' '!=' 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:29.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:29.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_13 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_13 ']' 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-19T20:55:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-19T20:55:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-19T20:55:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:30.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:30.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-19T20:55:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-19T20:55:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:30.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:30.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:30.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:30.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:30.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:30.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34813' '!=' 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:55:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_13 2026-02-19T20:55:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_13 ']' 2026-02-19T20:55:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-19T20:55:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-19T20:55:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:31.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-19T20:55:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-19T20:55:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:31.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-19T20:55:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-19T20:55:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:31.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:31.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:31.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:31.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:31.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:31.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:31.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:31.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:31.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-19T20:55:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-19T20:55:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:31.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:31.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:31.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:31.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34813' '!=' 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:31.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:31.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_13 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_13 ']' 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:33.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:33.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:33.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:33.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:33.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:33.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-19T20:55:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:33.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-19T20:55:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-19T20:55:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:33.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:33.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:33.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-19T20:55:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-19T20:55:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:33.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34813' '!=' 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:55:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_13 2026-02-19T20:55:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_13 ']' 2026-02-19T20:55:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-19T20:55:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-19T20:55:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-19T20:55:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:35.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:35.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:35.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:35.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:35.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-19T20:55:35.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:35.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-19T20:55:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-19T20:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-19T20:55:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:35.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:35.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:35.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:35.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:35.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-19T20:55:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-19T20:55:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34813], [object_number=32, tag_tid=1, entry_tid=34812], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34813' '!=' 'object_number=33, tag_tid=1, entry_tid=34813' 2026-02-19T20:55:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:55:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_13 2026-02-19T20:55:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_13 ']' 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:37.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:37.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-19T20:55:37.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:37.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-19T20:55:37.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:37.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-19T20:55:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:37.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-19T20:55:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-19T20:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-19T20:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:37.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:37.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:37.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:37.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:37.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:37.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:37.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-19T20:55:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-19T20:55:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T20:55:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-19T20:55:37.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-19T20:55:37.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-19T20:55:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:55:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:55:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34811], [object_number=34, tag_tid=1, entry_tid=34810]]], state=connected] 2026-02-19T20:55:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.status 2026-02-19T20:55:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:55:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2026-02-19T20:55:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:55:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:55:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:55:38.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2026-02-19T20:55:38.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_13': 2026-02-19T20:55:38.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:55:38.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:55:38.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:55:38.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a6c7492064 2026-02-19T20:55:38.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a6c7492064 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:45:11 2026 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:45:11 2026 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:46:12 2026 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a6c7492064 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: bb0fbbe0-269d-40de-85ab-495afa682240 2026-02-19T20:55:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:55:38.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:38.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_13 snap 2026-02-19T20:55:38.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T20:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.export 2026-02-19T20:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_13.export 2026-02-19T20:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_13.export 2026-02-19T20:55:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.export 2026-02-19T20:55:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:55:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_13.export 2026-02-19T20:55:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:55:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_13.export 2026-02-19T20:55:39.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_13.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_13.export 2026-02-19T20:55:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:55:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_14 2026-02-19T20:55:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_14 snap 2026-02-19T20:55:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:55:41.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2026-02-19T20:55:41.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2026-02-19T20:55:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:55:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:55:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:55:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2026-02-19T20:55:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:55:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:55:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:55:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2026-02-19T20:55:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:55:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:55:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:55:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:55:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:55:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:42.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:55:42.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2026-02-19T20:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:55:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:55:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:55:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2026-02-19T20:55:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:55:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2026-02-19T20:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:55:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:55:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2026-02-19T20:55:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:55:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_14 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_14 2026-02-19T20:55:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:55:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_14 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:55:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:55:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:55:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:55:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:43.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_14 2026-02-19T20:55:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:55:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:55:43.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:55:43.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:55:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:55:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:55:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:55:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_14 2026-02-19T20:55:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:55:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:55:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:55:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_14 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:55:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_14 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_14 ']' 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:43.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:43.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:43.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:43.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:43.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:43.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:43.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:43.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:43.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:43.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:43.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:43.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:43.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-19T20:55:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-19T20:55:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:55:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:55:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:55:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:55:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:55:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:55:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:55:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:55:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:55:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:55:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:55:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:55:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:43.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:43.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:43.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=50546], [object_number=49, tag_tid=1, entry_tid=50545], [object_number=48, tag_tid=1, entry_tid=50544], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:43.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:55:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-19T20:55:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-19T20:55:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=50546], [object_number=49, tag_tid=1, entry_tid=50545], [object_number=48, tag_tid=1, entry_tid=50544], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:43.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=50546' '!=' '' 2026-02-19T20:55:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:55:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=50546' = '' 2026-02-19T20:55:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:55:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:55:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:55:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:55:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:43.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:55:43.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:55:43.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:55:43.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:43.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:55:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:55:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=50546 2026-02-19T20:55:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 50546 2026-02-19T20:55:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:55:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:55:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_14 2026-02-19T20:55:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_14 ']' 2026-02-19T20:55:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-19T20:55:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-19T20:55:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:43.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-19T20:55:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:44.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:44.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:44.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:44.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:44.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:44.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-19T20:55:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:44.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:44.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:44.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:44.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:44.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:44.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-19T20:55:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:44.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-19T20:55:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:55:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:55:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:55:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:55:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:55:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:55:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:55:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:55:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:44.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-19T20:55:44.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:44.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:44.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:44.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-19T20:55:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:44.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:44.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:44.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:44.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:44.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=50546], [object_number=49, tag_tid=1, entry_tid=50545], [object_number=48, tag_tid=1, entry_tid=50544], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:55:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-19T20:55:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-19T20:55:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:44.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:44.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:44.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:44.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=50546], [object_number=49, tag_tid=1, entry_tid=50545], [object_number=48, tag_tid=1, entry_tid=50544], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=50546' '!=' 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:55:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_14 2026-02-19T20:55:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_14 ']' 2026-02-19T20:55:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-19T20:55:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-19T20:55:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-19T20:55:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:44.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:44.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:44.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:44.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-19T20:55:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:44.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:44.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:44.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-19T20:55:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:44.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:44.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:44.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:44.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:55:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:55:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:55:45.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:45.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:45.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:45.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:45.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=50546], [object_number=49, tag_tid=1, entry_tid=50545], [object_number=48, tag_tid=1, entry_tid=50544], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=50546], [object_number=49, tag_tid=1, entry_tid=50545], [object_number=48, tag_tid=1, entry_tid=50544], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=50546' '!=' 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:55:45.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_14 2026-02-19T20:55:45.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:45.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_14 ']' 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:45.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:46.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:46.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:46.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:46.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:46.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:46.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:55:46.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:55:46.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:55:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-19T20:55:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-19T20:55:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:46.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:46.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:46.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:46.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:46.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=50546], [object_number=49, tag_tid=1, entry_tid=50545], [object_number=48, tag_tid=1, entry_tid=50544], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:46.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:55:46.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-19T20:55:46.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:46.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:46.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-19T20:55:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:46.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:46.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:46.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:46.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:46.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=50546], [object_number=49, tag_tid=1, entry_tid=50545], [object_number=48, tag_tid=1, entry_tid=50544], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:46.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=50546' '!=' 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:55:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_14 2026-02-19T20:55:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_14 ']' 2026-02-19T20:55:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-19T20:55:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-19T20:55:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-19T20:55:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-19T20:55:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:48.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:48.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:48.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:48.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:48.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:48.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:48.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:48.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:48.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:48.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:48.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:48.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:48.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:48.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:48.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-19T20:55:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:55:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:55:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:55:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:55:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:55:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:55:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:48.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:48.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-19T20:55:48.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:48.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:48.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:48.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-19T20:55:48.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:48.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:48.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:48.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:48.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=50546], [object_number=49, tag_tid=1, entry_tid=50545], [object_number=48, tag_tid=1, entry_tid=50544], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:55:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-19T20:55:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-19T20:55:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=50546], [object_number=49, tag_tid=1, entry_tid=50545], [object_number=48, tag_tid=1, entry_tid=50544], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=50546' '!=' 'object_number=50, tag_tid=1, entry_tid=50546' 2026-02-19T20:55:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:55:49.398 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T20:55:49.399 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T20:55:49.399 INFO:tasks.rbd_mirror_thrash:waiting for 7.4989646963114565 secs before reviving daemons 2026-02-19T20:55:49.399 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T20:55:49.384+0000 7f61cbb32640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 86360) UID: 1000 2026-02-19T20:55:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_14 2026-02-19T20:55:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_14 ']' 2026-02-19T20:55:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-19T20:55:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-19T20:55:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-19T20:55:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:50.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:50.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:50.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:50.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T20:55:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-19T20:55:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:55:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:55:50.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:50.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:50.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:55:50.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:50.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-19T20:55:50.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:50.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-19T20:55:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:55:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:55:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:55:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:55:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:55:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:55:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:55:50.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:50.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:50.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:55:50.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-19T20:55:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-19T20:55:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-19T20:55:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:50.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:50.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:50.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:50.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:50.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:50.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:55:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-19T20:55:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-19T20:55:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T20:55:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-19T20:55:50.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-19T20:55:50.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:50.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:50.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-19T20:55:50.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:50.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 12 2026-02-19T20:55:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 12 2026-02-19T20:55:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=50546], [object_number=51, tag_tid=1, entry_tid=50543]]], state=connected] 2026-02-19T20:55:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.status 2026-02-19T20:55:50.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:55:50.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-19T20:55:50.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:50.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2026-02-19T20:55:50.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:50.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:50.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:50.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:50.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:55:50.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:55:50.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:55:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2026-02-19T20:55:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_14': 2026-02-19T20:55:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 117c804ac624 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.117c804ac624 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:45:24 2026 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:45:24 2026 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:46:25 2026 2026-02-19T20:55:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:55:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 117c804ac624 2026-02-19T20:55:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:55:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:55:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: d345ba20-24e6-4022-8fa4-2737b8391ecf 2026-02-19T20:55:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:55:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_14 snap 2026-02-19T20:55:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T20:55:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:55:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.export 2026-02-19T20:55:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_14.export 2026-02-19T20:55:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_14.export 2026-02-19T20:55:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.export 2026-02-19T20:55:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:55:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_14.export 2026-02-19T20:55:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:55:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_14.export 2026-02-19T20:55:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_14.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_14.export 2026-02-19T20:55:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:55:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_15 2026-02-19T20:55:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_15 snap 2026-02-19T20:55:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:52.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:55:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:55:52.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:55:55.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2026-02-19T20:55:55.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:55.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:55.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:55:55.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2026-02-19T20:55:55.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:55.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:55.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:55:55.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:55:55.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:55:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:55:56.898 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T20:55:56.898 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:55:56.898 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T20:55:56.898 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T20:55:56.898 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T20:55:56.900 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T20:55:56.900 INFO:tasks.rbd_mirror_thrash:waiting for 72.7824196973808 secs before thrashing 2026-02-19T20:55:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2026-02-19T20:55:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:55:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:55:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:55:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:55:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2026-02-19T20:55:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:55:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:55:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:55:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:55:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:56.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:55:56.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:56.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:55:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2026-02-19T20:55:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:55:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:55:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:55:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:56.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:56.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:56.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:56.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2026-02-19T20:55:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-19T20:55:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:55:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15 2026-02-19T20:55:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:55:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:55:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:55:56.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:55:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15 2026-02-19T20:55:57.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:55:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:55:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:55:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:55:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:55:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_15 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_15 2026-02-19T20:55:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:55:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:55:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:55:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:55:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:55:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:55:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:55:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:55:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:55:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_15 2026-02-19T20:55:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:55:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_15 ']' 2026-02-19T20:55:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-19T20:55:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-19T20:55:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:57.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:57.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:57.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:57.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:57.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:57.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:57.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:57.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:57.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:57.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-19T20:55:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-19T20:55:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-19T20:55:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:57.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:57.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:57.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:57.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-19T20:55:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-19T20:55:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-19T20:55:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:55:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:55:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:55:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:55:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:55:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:55:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:55:57.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:55:57.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:55:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:55:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:55:57.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:55:57.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:57.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:55:57.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:55:57.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=47472' '!=' '' 2026-02-19T20:55:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:55:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=47472' = '' 2026-02-19T20:55:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:55:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:55:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:55:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:55:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:55:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:55:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:57.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:55:57.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:55:57.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47472 2026-02-19T20:55:57.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 47472 2026-02-19T20:55:57.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:55:57.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:57.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:55:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_15 2026-02-19T20:55:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:55:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_15 ']' 2026-02-19T20:55:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-19T20:55:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-19T20:55:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:57.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:57.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:57.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-19T20:55:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-19T20:55:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-19T20:55:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:57.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:57.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:57.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:57.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:57.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-19T20:55:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-19T20:55:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-19T20:55:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:55:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:55:57.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:57.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:55:57.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:55:57.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:55:57.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:57.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:55:57.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:55:57.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:55:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-19T20:55:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-19T20:55:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:55:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:55:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:55:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:55:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:55:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:55:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:55:57.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=47472' '!=' 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:55:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_15 2026-02-19T20:55:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:55:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_15 ']' 2026-02-19T20:55:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-19T20:55:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-19T20:55:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-19T20:55:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:58.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:58.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-19T20:55:58.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-19T20:55:58.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:58.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:58.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:58.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-19T20:55:58.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:58.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:58.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-19T20:55:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-19T20:55:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-19T20:55:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:58.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:58.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:58.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:58.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:55:58.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:55:58.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:55:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:55:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:55:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:55:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:55:58.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:55:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-19T20:55:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-19T20:55:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:55:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:55:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:55:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:55:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:55:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:55:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:55:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=47472' '!=' 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:55:59.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_15 2026-02-19T20:55:59.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:55:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:55:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:55:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_15 ']' 2026-02-19T20:55:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-19T20:55:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-19T20:55:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:55:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:55:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:55:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:55:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:55:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:55:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:55:59.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:55:59.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:59.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-19T20:55:59.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:55:59.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:59.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:59.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:55:59.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:55:59.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:55:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:55:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:55:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:55:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:55:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:55:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:55:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:55:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-19T20:55:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-19T20:55:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:55:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:55:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:55:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-19T20:55:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:55:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:55:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:55:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:55:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:55:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:55:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:55:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:55:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:55:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-19T20:55:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:55:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-19T20:55:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-19T20:55:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:55:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:55:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:55:59.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:55:59.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:55:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:55:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:55:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:55:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:59.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:55:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-19T20:55:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:59.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:59.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-19T20:55:59.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:55:59.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:55:59.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:55:59.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:55:59.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:55:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:55:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:55:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:59.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:55:59.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:55:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:55:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:55:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:55:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:55:59.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:55:59.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:59.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:59.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=47472' '!=' 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:55:59.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:55:59.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:56:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_15 2026-02-19T20:56:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:56:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_15 ']' 2026-02-19T20:56:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-19T20:56:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-19T20:56:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-19T20:56:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:01.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-19T20:56:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-19T20:56:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:01.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:01.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:01.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-19T20:56:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-19T20:56:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-19T20:56:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:56:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:56:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:56:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:56:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:56:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:56:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:56:01.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:56:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-19T20:56:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-19T20:56:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:56:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:56:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:56:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:56:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:56:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:56:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:56:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:56:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:56:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=47472' '!=' 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:56:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:56:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_15 2026-02-19T20:56:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:56:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_15 ']' 2026-02-19T20:56:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-19T20:56:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-19T20:56:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-19T20:56:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-19T20:56:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-19T20:56:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:56:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-19T20:56:03.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-19T20:56:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-19T20:56:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:56:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:56:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:56:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:03.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:56:03.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:56:03.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:03.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:56:03.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:56:03.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:56:03.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-19T20:56:03.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:03.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:03.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:03.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-19T20:56:03.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:03.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:03.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:03.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:03.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:56:03.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:56:03.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:03.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:03.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:56:03.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:56:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:56:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:56:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=47472], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470], [object_number=45, tag_tid=1, entry_tid=47469]]], state=connected] 2026-02-19T20:56:03.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:56:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:56:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=47472' '!=' 'object_number=44, tag_tid=1, entry_tid=47472' 2026-02-19T20:56:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:56:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_15 2026-02-19T20:56:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:56:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_15 ']' 2026-02-19T20:56:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-19T20:56:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-19T20:56:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:05.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:05.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:05.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-19T20:56:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-19T20:56:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-19T20:56:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:05.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:05.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:05.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:05.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:56:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:56:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:56:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:05.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:56:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:56:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:56:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:56:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:05.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-19T20:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-19T20:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T20:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-19T20:56:05.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-19T20:56:05.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-19T20:56:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:56:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:56:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:56:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47471], [object_number=46, tag_tid=1, entry_tid=47470]]], state=connected] 2026-02-19T20:56:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.status 2026-02-19T20:56:06.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:56:06.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:56:06.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:06.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2026-02-19T20:56:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:56:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:56:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:56:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:56:07.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2026-02-19T20:56:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_15': 2026-02-19T20:56:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:56:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:56:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:56:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11f2f7ef7373 2026-02-19T20:56:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11f2f7ef7373 2026-02-19T20:56:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:56:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:56:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:56:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:56:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:45:35 2026 2026-02-19T20:56:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:45:35 2026 2026-02-19T20:56:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:48:01 2026 2026-02-19T20:56:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:56:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11f2f7ef7373 2026-02-19T20:56:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:56:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:56:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 46003d68-84ae-4ffa-9988-437fc494a9e0 2026-02-19T20:56:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:56:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_15 snap 2026-02-19T20:56:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T20:56:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:56:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.export 2026-02-19T20:56:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_15.export 2026-02-19T20:56:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_15.export 2026-02-19T20:56:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.export 2026-02-19T20:56:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:56:07.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_15.export 2026-02-19T20:56:07.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:56:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_15.export 2026-02-19T20:56:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_15.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_15.export 2026-02-19T20:56:07.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:07.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:56:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_16 2026-02-19T20:56:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_16 snap 2026-02-19T20:56:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:56:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:56:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2026-02-19T20:56:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:56:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2026-02-19T20:56:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:56:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:56:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:56:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:56:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2026-02-19T20:56:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:56:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:56:10.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:56:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:56:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:56:10.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:10.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:10.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:10.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:10.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-02-19T20:56:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:10.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:10.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2026-02-19T20:56:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:56:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:56:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:56:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:56:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2026-02-19T20:56:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:56:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:56:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:11.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:11.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:11.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2026-02-19T20:56:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_16 2026-02-19T20:56:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:56:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:56:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:56:11.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:11.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:11.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:11.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_16 2026-02-19T20:56:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:11.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 1 2026-02-19T20:56:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:56:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:56:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:56:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-02-19T20:56:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2026-02-19T20:56:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:56:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:56:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:12.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:12.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:12.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2026-02-19T20:56:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:12.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2026-02-19T20:56:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:56:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:56:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:56:12.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2026-02-19T20:56:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_16 2026-02-19T20:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:56:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:56:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:56:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_16 2026-02-19T20:56:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:12.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 2 2026-02-19T20:56:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:56:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-02-19T20:56:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:56:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:56:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:56:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:56:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:56:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-02-19T20:56:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2026-02-19T20:56:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:56:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:56:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:56:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:56:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2026-02-19T20:56:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2026-02-19T20:56:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:56:14.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:14.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:56:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:56:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:56:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:56:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:14.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:14.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:14.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:14.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2026-02-19T20:56:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_16 2026-02-19T20:56:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:56:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:56:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:14.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:56:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:56:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:56:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:56:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:56:14.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:14.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:14.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:14.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_16 2026-02-19T20:56:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 4 2026-02-19T20:56:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:56:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-02-19T20:56:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:56:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:56:18.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-02-19T20:56:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2026-02-19T20:56:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:56:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:56:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:56:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:56:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:56:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2026-02-19T20:56:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2026-02-19T20:56:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:56:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:56:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:18.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:56:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:56:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2026-02-19T20:56:18.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_16 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:56:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:18.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:56:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:56:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:56:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:56:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:56:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:19.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:56:19.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_16 2026-02-19T20:56:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-19T20:56:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 8 2026-02-19T20:56:27.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:27.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:27.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:56:27.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:56:27.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:27.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:27.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:27.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:27.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-02-19T20:56:27.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:56:27.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:56:27.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:56:27.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:56:27.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:56:27.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:56:27.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:56:27.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_16 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_16 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:27.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_16 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_16 ']' 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:27.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:27.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:27.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:27.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-19T20:56:27.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:27.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-19T20:56:27.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:27.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_16 2026-02-19T20:56:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-19T20:56:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_16 2026-02-19T20:56:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-19T20:56:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-19T20:56:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:27.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.status 2026-02-19T20:56:27.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-19T20:56:27.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.status 2026-02-19T20:56:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:56:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:56:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46043], [object_number=46, tag_tid=1, entry_tid=46042]]], state=connected] 2026-02-19T20:56:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46043], [object_number=46, tag_tid=1, entry_tid=46042]]], state=connected] 2026-02-19T20:56:27.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.status 2026-02-19T20:56:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:56:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-19T20:56:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_16 2026-02-19T20:56:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-19T20:56:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:27.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_16 2026-02-19T20:56:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-19T20:56:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-19T20:56:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:27.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.status 2026-02-19T20:56:27.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-19T20:56:27.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.status 2026-02-19T20:56:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:56:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:56:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46043], [object_number=46, tag_tid=1, entry_tid=46042]]], state=connected] 2026-02-19T20:56:27.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46043], [object_number=46, tag_tid=1, entry_tid=46042]]], state=connected] 2026-02-19T20:56:27.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.status 2026-02-19T20:56:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:56:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:56:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2026-02-19T20:56:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:56:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:56:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:56:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:56:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2026-02-19T20:56:28.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_16': 2026-02-19T20:56:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:56:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:56:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:56:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11f2e6ce905a 2026-02-19T20:56:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11f2e6ce905a 2026-02-19T20:56:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:56:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:56:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:56:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:56:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:45:48 2026 2026-02-19T20:56:28.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:45:48 2026 2026-02-19T20:56:28.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:48:01 2026 2026-02-19T20:56:28.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:56:28.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11f2e6ce905a 2026-02-19T20:56:28.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:56:28.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:56:28.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 05cd9384-5c25-4685-b4ba-e437f9e9cb8d 2026-02-19T20:56:28.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:56:28.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_16 snap 2026-02-19T20:56:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T20:56:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:56:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.export 2026-02-19T20:56:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_16.export 2026-02-19T20:56:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_16.export 2026-02-19T20:56:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.export 2026-02-19T20:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:56:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_16.export 2026-02-19T20:56:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:56:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_16.export 2026-02-19T20:56:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_16.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_16.export 2026-02-19T20:56:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:56:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_17 2026-02-19T20:56:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_17 snap 2026-02-19T20:56:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:28.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:28.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:28.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:56:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2026-02-19T20:56:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:56:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:56:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:56:32.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2026-02-19T20:56:32.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:56:32.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:56:32.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:32.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:32.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:56:32.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:56:32.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:32.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:56:32.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:32.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:56:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:56:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:32.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:32.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:32.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:32.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2026-02-19T20:56:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17 2026-02-19T20:56:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:56:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:56:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:56:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:56:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17 2026-02-19T20:56:32.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:32.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:32.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_17 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:56:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:56:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_17 2026-02-19T20:56:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:56:33.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:56:33.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:56:33.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:56:33.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:56:33.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:56:33.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:56:33.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:33.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_17 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_17 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:33.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:56:33.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:56:33.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:56:33.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:33.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_17 2026-02-19T20:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_17 ']' 2026-02-19T20:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-19T20:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-19T20:56:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-19T20:56:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:33.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:33.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:33.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:33.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:56:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:33.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:33.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:33.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-02-19T20:56:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:56:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:33.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:33.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:33.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-19T20:56:33.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:33.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:33.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-19T20:56:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-19T20:56:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-19T20:56:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-19T20:56:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:56:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:56:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770], [object_number=37, tag_tid=1, entry_tid=38769], [object_number=36, tag_tid=1, entry_tid=38768]]], state=connected] 2026-02-19T20:56:33.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:33.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:56:33.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-19T20:56:33.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-19T20:56:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-19T20:56:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-19T20:56:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-19T20:56:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:56:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:56:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770], [object_number=37, tag_tid=1, entry_tid=38769], [object_number=36, tag_tid=1, entry_tid=38768]]], state=connected] 2026-02-19T20:56:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38771' '!=' '' 2026-02-19T20:56:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:56:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38771' = '' 2026-02-19T20:56:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:56:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:56:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:56:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:56:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:56:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:56:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:56:33.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:56:33.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:33.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:56:33.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38771 2026-02-19T20:56:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 38771 2026-02-19T20:56:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:56:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:56:33.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_17 2026-02-19T20:56:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_17 ']' 2026-02-19T20:56:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-19T20:56:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-19T20:56:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:33.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:33.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:33.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:56:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:33.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:33.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:33.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:34.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:34.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-19T20:56:34.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:34.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:34.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:34.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-19T20:56:34.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:34.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:34.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:34.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:34.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-19T20:56:34.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-19T20:56:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-19T20:56:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:56:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:56:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770], [object_number=37, tag_tid=1, entry_tid=38769], [object_number=36, tag_tid=1, entry_tid=38768]]], state=connected] 2026-02-19T20:56:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:56:34.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-19T20:56:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-19T20:56:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:56:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:56:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770], [object_number=37, tag_tid=1, entry_tid=38769], [object_number=36, tag_tid=1, entry_tid=38768]]], state=connected] 2026-02-19T20:56:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38771' '!=' 'object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:56:34.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_17 2026-02-19T20:56:34.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:34.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:34.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:34.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:34.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_17 ']' 2026-02-19T20:56:34.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-19T20:56:34.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-19T20:56:34.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:34.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:34.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:34.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:34.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:34.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:34.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:34.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:34.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:34.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:34.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-19T20:56:34.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:34.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:34.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:34.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:34.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:34.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:34.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:34.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:34.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:34.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:34.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:34.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:34.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:34.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-19T20:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:34.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:34.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:34.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:34.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-02-19T20:56:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:56:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:56:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:56:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:34.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:34.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:56:34.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:34.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:34.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:56:34.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:34.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:34.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:34.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:34.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:34.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:34.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:34.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:34.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:34.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:34.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:34.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:34.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:34.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:34.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:34.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:34.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:34.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-19T20:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-19T20:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-19T20:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-19T20:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-19T20:56:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770], [object_number=37, tag_tid=1, entry_tid=38769], [object_number=36, tag_tid=1, entry_tid=38768]]], state=connected] 2026-02-19T20:56:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-19T20:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-19T20:56:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-19T20:56:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-19T20:56:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-19T20:56:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:56:34.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:56:34.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:34.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:34.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770], [object_number=37, tag_tid=1, entry_tid=38769], [object_number=36, tag_tid=1, entry_tid=38768]]], state=connected] 2026-02-19T20:56:34.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:34.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:34.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:34.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38771' '!=' 'object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:34.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:34.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:56:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_17 2026-02-19T20:56:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_17 ']' 2026-02-19T20:56:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-19T20:56:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-19T20:56:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:35.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:35.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:35.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-19T20:56:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:56:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:56:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:35.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:35.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:35.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:35.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-02-19T20:56:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:56:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:56:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:35.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:35.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:35.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:35.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-19T20:56:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-19T20:56:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-19T20:56:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-19T20:56:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-19T20:56:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:35.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:56:35.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:56:35.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:35.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:35.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770], [object_number=37, tag_tid=1, entry_tid=38769], [object_number=36, tag_tid=1, entry_tid=38768]]], state=connected] 2026-02-19T20:56:35.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:35.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:56:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-19T20:56:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-19T20:56:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:35.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-19T20:56:35.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-19T20:56:35.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:35.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:35.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-19T20:56:35.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:56:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:56:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770], [object_number=37, tag_tid=1, entry_tid=38769], [object_number=36, tag_tid=1, entry_tid=38768]]], state=connected] 2026-02-19T20:56:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38771' '!=' 'object_number=39, tag_tid=1, entry_tid=38771' 2026-02-19T20:56:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:56:37.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_17 2026-02-19T20:56:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_17 ']' 2026-02-19T20:56:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-19T20:56:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-19T20:56:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:37.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:37.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-19T20:56:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:56:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:56:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:37.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:37.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:37.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:37.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:37.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:37.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:37.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:37.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-19T20:56:37.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:37.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2026-02-19T20:56:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:56:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:56:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:37.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:37.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:37.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2026-02-19T20:56:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:37.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-19T20:56:37.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:37.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:37.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:37.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-19T20:56:37.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:37.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:37.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:37.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:37.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-19T20:56:37.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-19T20:56:37.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-19T20:56:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:56:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:56:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:56:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-19T20:56:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-19T20:56:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:37.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:56:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:56:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38771], [object_number=38, tag_tid=1, entry_tid=38770]]], state=connected] 2026-02-19T20:56:37.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.status 2026-02-19T20:56:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:56:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:56:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2026-02-19T20:56:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:56:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:56:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:56:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2026-02-19T20:56:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_17': 2026-02-19T20:56:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:56:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11f2ec5c9728 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11f2ec5c9728 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:46:01 2026 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:46:01 2026 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:48:03 2026 2026-02-19T20:56:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:56:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11f2ec5c9728 2026-02-19T20:56:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:56:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:56:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: eac262df-4bca-4f08-9721-bb777cecfa34 2026-02-19T20:56:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:56:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_17 snap 2026-02-19T20:56:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T20:56:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:56:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.export 2026-02-19T20:56:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_17.export 2026-02-19T20:56:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_17.export 2026-02-19T20:56:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.export 2026-02-19T20:56:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:56:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_17.export 2026-02-19T20:56:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:56:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_17.export 2026-02-19T20:56:39.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_17.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_17.export 2026-02-19T20:56:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:56:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_18 2026-02-19T20:56:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_18 snap 2026-02-19T20:56:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:56:42.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2026-02-19T20:56:42.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:42.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:42.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:42.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2026-02-19T20:56:42.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:42.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:42.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:42.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:56:42.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:56:42.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:42.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:56:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2026-02-19T20:56:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:56:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:56:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:56:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2026-02-19T20:56:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:56:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:56:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:56:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:56:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:56:43.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:56:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:56:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:56:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:43.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2026-02-19T20:56:43.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:56:43.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:56:43.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:56:43.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:56:43.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:56:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:56:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:56:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_18 2026-02-19T20:56:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:56:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_18 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:56:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:56:43.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:43.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:56:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_18 2026-02-19T20:56:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_18 ']' 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-19T20:56:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:44.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:44.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:44.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-19T20:56:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-19T20:56:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-19T20:56:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:44.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:44.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:44.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:44.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:44.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:44.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:44.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-19T20:56:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-19T20:56:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:44.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:44.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:44.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:44.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:44.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=31701' '!=' '' 2026-02-19T20:56:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:56:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=31701' = '' 2026-02-19T20:56:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:56:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:56:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:56:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:56:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:56:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:56:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:56:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31701 2026-02-19T20:56:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 31701 2026-02-19T20:56:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:56:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:56:44.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_18 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_18 ']' 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:44.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:44.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:44.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:44.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:44.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:44.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:44.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:44.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:44.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:44.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:44.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:44.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-19T20:56:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:44.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:44.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:44.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:44.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-19T20:56:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-19T20:56:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:44.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=31701' '!=' 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:56:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_18 2026-02-19T20:56:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_18 ']' 2026-02-19T20:56:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-19T20:56:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-19T20:56:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:45.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:45.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:45.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:45.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:45.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:45.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-19T20:56:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-19T20:56:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-19T20:56:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-19T20:56:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-19T20:56:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:45.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=31701' '!=' 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_18 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_18 ']' 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:46.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:46.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:46.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:46.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-19T20:56:46.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-19T20:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-19T20:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:46.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:46.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:46.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:46.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:46.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:46.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:46.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:46.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:46.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-19T20:56:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-19T20:56:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:46.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:46.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:46.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:46.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=31701' '!=' 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:56:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_18 2026-02-19T20:56:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_18 ']' 2026-02-19T20:56:47.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-19T20:56:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-19T20:56:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:47.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:47.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:47.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:47.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:47.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:47.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:47.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:47.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:47.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:47.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:47.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:47.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-19T20:56:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-19T20:56:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-19T20:56:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-19T20:56:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-19T20:56:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=31701' '!=' 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:56:48.226 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.34758359653726256) > weight (0.25) 2026-02-19T20:56:48.226 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-19T20:56:48.226 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-19T20:56:48.226 INFO:tasks.rbd_mirror_thrash:waiting for 8.652629637798384 secs before reviving daemons 2026-02-19T20:56:48.227 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T20:56:48.216+0000 7feb3576a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 93158) UID: 1000 2026-02-19T20:56:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_18 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_18 ']' 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-19T20:56:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:50.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:50.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-19T20:56:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-19T20:56:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:50.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-19T20:56:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:50.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:50.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-19T20:56:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-19T20:56:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:50.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:50.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:50.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:50.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=31701], [object_number=28, tag_tid=1, entry_tid=31700], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:50.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=31701' '!=' 'object_number=29, tag_tid=1, entry_tid=31701' 2026-02-19T20:56:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:56:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_18 2026-02-19T20:56:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_18 ']' 2026-02-19T20:56:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-19T20:56:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-19T20:56:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-19T20:56:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:52.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-19T20:56:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-19T20:56:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-19T20:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:52.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:52.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:52.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:52.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:52.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:52.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:52.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:52.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-19T20:56:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-19T20:56:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T20:56:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-19T20:56:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-19T20:56:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:52.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-19T20:56:52.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:56:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:56:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31699], [object_number=30, tag_tid=1, entry_tid=31698]]], state=connected] 2026-02-19T20:56:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.status 2026-02-19T20:56:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:56:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2026-02-19T20:56:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:56:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:56:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:56:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2026-02-19T20:56:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_18': 2026-02-19T20:56:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:56:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:56:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:56:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11f28e8f175f 2026-02-19T20:56:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11f28e8f175f 2026-02-19T20:56:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:56:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:56:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:56:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:56:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:46:12 2026 2026-02-19T20:56:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:46:12 2026 2026-02-19T20:56:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:48:14 2026 2026-02-19T20:56:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:56:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11f28e8f175f 2026-02-19T20:56:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:56:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:56:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 14c5ec4a-5f0d-447a-99bb-910375b600d3 2026-02-19T20:56:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:56:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_18 snap 2026-02-19T20:56:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T20:56:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:56:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.export 2026-02-19T20:56:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_18.export 2026-02-19T20:56:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_18.export 2026-02-19T20:56:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.export 2026-02-19T20:56:53.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:56:53.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_18.export 2026-02-19T20:56:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:56:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_18.export 2026-02-19T20:56:54.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_18.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_18.export 2026-02-19T20:56:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:56:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_19 2026-02-19T20:56:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_19 snap 2026-02-19T20:56:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:56:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:56:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:56:56.880 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-19T20:56:56.880 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:56:56.881 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-19T20:56:56.881 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-19T20:56:56.881 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-19T20:56:56.884 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-19T20:56:56.884 INFO:tasks.rbd_mirror_thrash:waiting for 87.61312797055783 secs before thrashing 2026-02-19T20:56:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2026-02-19T20:56:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:56:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2026-02-19T20:56:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:56:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:56:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:56:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:56:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2026-02-19T20:56:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:56:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:56:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:56:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:56:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2026-02-19T20:56:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:56:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:56:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:56:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:56:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:56:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:58.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:56:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2026-02-19T20:56:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:56:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:56:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:56:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:58.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2026-02-19T20:56:58.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-19T20:56:58.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:58.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:58.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:58.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19 2026-02-19T20:56:58.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:56:58.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:58.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:56:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19 2026-02-19T20:56:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-19T20:56:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:56:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_19 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:56:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:56:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:56:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:56:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:56:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:56:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_19 2026-02-19T20:56:59.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:56:59.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:56:59.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:56:59.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:56:59.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:56:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:56:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:56:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_19 2026-02-19T20:56:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:56:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:56:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:56:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:56:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:56:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_19 2026-02-19T20:56:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:56:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:56:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:56:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:56:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:56:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:56:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:56:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:56:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:56:59.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_19 2026-02-19T20:56:59.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:56:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_19 ']' 2026-02-19T20:56:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-19T20:56:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-19T20:56:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:59.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:59.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:59.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-19T20:56:59.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:59.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:59.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:59.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:59.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-19T20:56:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-19T20:56:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-19T20:56:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-19T20:56:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:56:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:59.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-19T20:56:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:56:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-19T20:56:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:56:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-19T20:56:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:56:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:56:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-19T20:56:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-19T20:56:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:56:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-19T20:56:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:56:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:56:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:56:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:59.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198]]], state=connected] 2026-02-19T20:56:59.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=25200], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198], [object_number=21, tag_tid=1, entry_tid=25197]]], state=connected] 2026-02-19T20:56:59.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:56:59.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:56:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-19T20:56:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:56:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-19T20:56:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:56:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:56:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:56:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:56:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-19T20:56:59.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-19T20:56:59.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:56:59.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:56:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-19T20:56:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:56:59.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:56:59.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:56:59.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:56:59.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198]]], state=connected] 2026-02-19T20:56:59.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=25200], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198], [object_number=21, tag_tid=1, entry_tid=25197]]], state=connected] 2026-02-19T20:56:59.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:56:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:56:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:56:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=25200' '!=' '' 2026-02-19T20:56:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:56:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=25200' = '' 2026-02-19T20:56:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:56:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:56:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:56:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:56:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:56:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:56:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:56:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:56:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:56:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:56:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:56:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:56:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:56:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=25200 2026-02-19T20:56:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 25200 2026-02-19T20:56:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:56:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:56:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:56:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_19 2026-02-19T20:56:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_19 ']' 2026-02-19T20:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-19T20:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-19T20:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:56:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:56:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-19T20:56:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:56:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:56:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:56:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:56:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:56:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:56:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:56:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:56:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:56:59.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:59.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:59.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:56:59.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-19T20:56:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-19T20:56:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-19T20:56:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:56:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:56:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:56:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:56:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:56:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:56:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:56:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:56:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:56:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:56:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:56:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:56:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:56:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:56:59.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:56:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-19T20:56:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-19T20:56:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:56:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:56:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:56:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-19T20:56:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:56:59.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:56:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:56:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:56:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:56:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:56:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:56:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:56:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:56:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:56:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-19T20:57:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-19T20:57:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-19T20:57:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-19T20:57:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-19T20:57:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-19T20:57:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198]]], state=connected] 2026-02-19T20:57:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=25200], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198], [object_number=21, tag_tid=1, entry_tid=25197]]], state=connected] 2026-02-19T20:57:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-19T20:57:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-19T20:57:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:00.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:00.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:00.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:00.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-19T20:57:00.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-19T20:57:00.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:00.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-19T20:57:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198]]], state=connected] 2026-02-19T20:57:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=25200], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198], [object_number=21, tag_tid=1, entry_tid=25197]]], state=connected] 2026-02-19T20:57:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:57:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:57:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=25200' '!=' 'object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:57:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:57:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_19 2026-02-19T20:57:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_19 ']' 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:00.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:00.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:00.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:00.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:00.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:00.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-19T20:57:00.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-19T20:57:00.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:00.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:00.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:00.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-19T20:57:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-19T20:57:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-19T20:57:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:57:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:57:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:57:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:57:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:57:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:57:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:57:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:57:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:57:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:57:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:57:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-19T20:57:00.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:00.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-19T20:57:00.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:00.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:00.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:00.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-19T20:57:00.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:00.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:00.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:00.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:00.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-19T20:57:00.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-19T20:57:00.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:00.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:00.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-19T20:57:00.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198]]], state=connected] 2026-02-19T20:57:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=25200], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198], [object_number=21, tag_tid=1, entry_tid=25197]]], state=connected] 2026-02-19T20:57:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-19T20:57:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-19T20:57:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-19T20:57:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-19T20:57:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-19T20:57:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:01.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:01.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:01.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:01.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198]]], state=connected] 2026-02-19T20:57:01.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=25200], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198], [object_number=21, tag_tid=1, entry_tid=25197]]], state=connected] 2026-02-19T20:57:01.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:57:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:57:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=25200' '!=' 'object_number=20, tag_tid=1, entry_tid=25200' 2026-02-19T20:57:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:57:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_19 2026-02-19T20:57:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:57:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_19 ']' 2026-02-19T20:57:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-19T20:57:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-19T20:57:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-19T20:57:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-19T20:57:01.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-19T20:57:01.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:01.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-19T20:57:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-19T20:57:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-19T20:57:01.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:01.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:57:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:57:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-19T20:57:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:02.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-19T20:57:02.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-19T20:57:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-19T20:57:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-19T20:57:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:02.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:02.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-19T20:57:02.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198]]], state=connected] 2026-02-19T20:57:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198]]], state=connected] 2026-02-19T20:57:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-19T20:57:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-19T20:57:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T20:57:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-19T20:57:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-19T20:57:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-19T20:57:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198]]], state=connected] 2026-02-19T20:57:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=25199], [object_number=22, tag_tid=1, entry_tid=25198]]], state=connected] 2026-02-19T20:57:02.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.status 2026-02-19T20:57:02.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:02.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:02.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:02.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2026-02-19T20:57:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:57:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:57:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:03.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2026-02-19T20:57:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_19': 2026-02-19T20:57:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:57:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:57:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:57:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11a2f9b3bdb8 2026-02-19T20:57:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11a2f9b3bdb8 2026-02-19T20:57:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:57:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:57:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:46:22 2026 2026-02-19T20:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:46:22 2026 2026-02-19T20:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:47:23 2026 2026-02-19T20:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11a2f9b3bdb8 2026-02-19T20:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 5e9a00a8-3bf9-47d2-a5f8-ca1733a2c5ef 2026-02-19T20:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_19 snap 2026-02-19T20:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T20:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.export 2026-02-19T20:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_19.export 2026-02-19T20:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_19.export 2026-02-19T20:57:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.export 2026-02-19T20:57:03.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:03.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_19.export 2026-02-19T20:57:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:03.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_19.export 2026-02-19T20:57:03.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_19.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_19.export 2026-02-19T20:57:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:57:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_20 2026-02-19T20:57:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_20 snap 2026-02-19T20:57:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:57:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2026-02-19T20:57:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2026-02-19T20:57:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:57:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2026-02-19T20:57:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:57:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:57:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:57:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2026-02-19T20:57:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:57:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:57:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:57:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:57:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:57:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:57:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:57:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:57:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:57:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:04.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:04.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2026-02-19T20:57:04.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:57:04.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:57:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:57:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:57:04.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:57:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:57:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:57:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_20 2026-02-19T20:57:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:57:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:57:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:57:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:57:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_20 2026-02-19T20:57:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:57:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:57:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:57:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:57:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:57:04.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:57:04.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:04.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:57:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_20 2026-02-19T20:57:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_20 ']' 2026-02-19T20:57:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-19T20:57:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-19T20:57:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:05.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:05.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-19T20:57:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-19T20:57:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-19T20:57:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-19T20:57:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-19T20:57:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-19T20:57:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:57:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:57:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2]]], state=connected] 2026-02-19T20:57:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2], [object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-19T20:57:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:57:05.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-19T20:57:05.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:05.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:05.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-19T20:57:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-19T20:57:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-19T20:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-19T20:57:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:57:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:57:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2]]], state=connected] 2026-02-19T20:57:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2], [object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-19T20:57:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:05.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=3' '!=' '' 2026-02-19T20:57:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:57:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=3' = '' 2026-02-19T20:57:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:57:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:57:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T20:57:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:57:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:05.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=2 2026-02-19T20:57:05.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:57:05.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:57:05.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:57:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:57:05.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:05.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=3 2026-02-19T20:57:05.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 2 -a 1 -le 3 2026-02-19T20:57:05.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:57:05.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:05.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:57:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_20 2026-02-19T20:57:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_20 ']' 2026-02-19T20:57:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-19T20:57:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-19T20:57:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:05.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:05.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:05.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-19T20:57:05.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:05.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:05.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:05.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:05.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:05.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:05.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:05.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-19T20:57:05.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:05.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-19T20:57:05.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:05.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:05.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:05.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-19T20:57:05.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-19T20:57:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-19T20:57:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-19T20:57:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:57:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:57:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2]]], state=connected] 2026-02-19T20:57:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2], [object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-19T20:57:05.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-19T20:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-19T20:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-19T20:57:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-19T20:57:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-19T20:57:05.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:57:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:57:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2]]], state=connected] 2026-02-19T20:57:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2], [object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-19T20:57:05.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=3' '!=' 'object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:57:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_20 2026-02-19T20:57:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_20 ']' 2026-02-19T20:57:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-19T20:57:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-19T20:57:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-19T20:57:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-19T20:57:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-19T20:57:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-19T20:57:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:57:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:57:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2]]], state=connected] 2026-02-19T20:57:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2], [object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-19T20:57:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:57:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-19T20:57:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-19T20:57:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-19T20:57:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-19T20:57:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-19T20:57:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:57:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:57:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2]]], state=connected] 2026-02-19T20:57:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2], [object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-19T20:57:06.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=3' '!=' 'object_number=19, tag_tid=2, entry_tid=3' 2026-02-19T20:57:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:57:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_20 2026-02-19T20:57:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_20 ']' 2026-02-19T20:57:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-19T20:57:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-19T20:57:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-19T20:57:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:07.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:07.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:07.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:07.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:07.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:07.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-19T20:57:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-19T20:57:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-19T20:57:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-19T20:57:07.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-19T20:57:07.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-19T20:57:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:07.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:57:07.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:57:07.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:07.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2]]], state=connected] 2026-02-19T20:57:07.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2]]], state=connected] 2026-02-19T20:57:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:57:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-19T20:57:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-19T20:57:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T20:57:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:07.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-19T20:57:07.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-19T20:57:07.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:07.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:07.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-19T20:57:07.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:57:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:57:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2]]], state=connected] 2026-02-19T20:57:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=19, tag_tid=2, entry_tid=3], [object_number=18, tag_tid=2, entry_tid=2]]], state=connected] 2026-02-19T20:57:07.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.status 2026-02-19T20:57:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:57:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=3, entry_tid=1' 2026-02-19T20:57:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2026-02-19T20:57:07.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:07.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:07.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:07.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:07.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:07.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:57:07.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2026-02-19T20:57:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_20': 2026-02-19T20:57:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:57:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:57:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11fcca0ae6ed 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11fcca0ae6ed 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:47:04 2026 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:47:04 2026 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:47:04 2026 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:57:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11fcca0ae6ed 2026-02-19T20:57:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:57:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:57:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 34aaf255-a028-4dda-90c0-ae0d48515507 2026-02-19T20:57:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:57:08.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_20 snap 2026-02-19T20:57:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T20:57:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:57:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.export 2026-02-19T20:57:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_20.export 2026-02-19T20:57:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_20.export 2026-02-19T20:57:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.export 2026-02-19T20:57:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_20.export 2026-02-19T20:57:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_20.export 2026-02-19T20:57:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_20.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_20.export 2026-02-19T20:57:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:57:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_21 2026-02-19T20:57:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_21 snap 2026-02-19T20:57:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:57:09.683 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6497696177809664) > weight (0.25) 2026-02-19T20:57:09.683 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-19T20:57:09.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-19T20:57:09.683 INFO:tasks.rbd_mirror_thrash:waiting for 3.4892354549437243 secs before reviving daemons 2026-02-19T20:57:09.684 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:57:09.672+0000 7f1f194f5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 91581) UID: 1000 2026-02-19T20:57:09.684 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:57:09.672+0000 7f1f11ce6640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55ea8fadc000 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-19T20:57:09.684 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T20:57:09.672+0000 7f1f11ce6640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55ea8fadc000 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-19T20:57:09.699 INFO:tasks.ceph.cluster1.osd.0.trial013.stderr:2026-02-19T20:57:09.694+0000 7f93a8cad640 -1 reset not still connected to 0x556e747aadd0 2026-02-19T20:57:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2026-02-19T20:57:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2026-02-19T20:57:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:57:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:12.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:13.173 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-19T20:57:13.174 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:57:13.174 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-19T20:57:13.174 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-19T20:57:13.174 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-19T20:57:13.176 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-19T20:57:13.177 INFO:tasks.rbd_mirror_thrash:waiting for 112.82434457548729 secs before thrashing 2026-02-19T20:57:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2026-02-19T20:57:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:57:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:57:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:57:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2026-02-19T20:57:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:57:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:57:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:57:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:57:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:57:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:57:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:57:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:57:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:57:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:57:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:57:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:57:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2026-02-19T20:57:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:57:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_21 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:57:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:57:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:57:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:57:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:57:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:13.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:13.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:13.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:13.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_21 2026-02-19T20:57:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:57:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_21 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:57:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:57:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:57:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:57:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:57:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_21 2026-02-19T20:57:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:57:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_21 2026-02-19T20:57:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:57:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:57:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:57:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:57:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:57:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:57:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_21 2026-02-19T20:57:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:57:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:57:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:57:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:57:14.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:57:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:57:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:57:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_21 2026-02-19T20:57:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:57:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_21 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:57:14.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:57:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:57:14.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_21 2026-02-19T20:57:14.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:14.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:14.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:14.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:14.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_21 ']' 2026-02-19T20:57:14.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-19T20:57:14.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-19T20:57:14.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:14.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:14.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-19T20:57:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:14.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:14.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:14.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:14.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:14.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-02-19T20:57:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:14.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2026-02-19T20:57:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:57:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:57:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:57:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:57:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:57:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:57:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:57:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:14.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_21 2026-02-19T20:57:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:57:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:57:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:57:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:57:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:57:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:57:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:57:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:57:14.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:57:14.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:14.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:14.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:14.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:57:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:57:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:57:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-19T20:57:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-19T20:57:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:14.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:14.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:14.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:14.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:14.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:14.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:14.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:14.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:14.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:14.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-19T20:57:14.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:14.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:14.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:14.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-19T20:57:14.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:14.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:14.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:14.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26546' '!=' '' 2026-02-19T20:57:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:57:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26546' = '' 2026-02-19T20:57:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:57:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:57:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:57:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:57:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:57:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:57:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:57:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:14.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=26546 2026-02-19T20:57:14.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 26546 2026-02-19T20:57:14.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:57:14.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:14.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:57:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_21 2026-02-19T20:57:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_21 ']' 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-19T20:57:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2026-02-19T20:57:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:57:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:57:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:57:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:57:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:57:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:57:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:15.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:15.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_21 2026-02-19T20:57:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:57:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:57:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:57:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:57:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:57:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-19T20:57:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-19T20:57:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:15.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:15.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:15.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:15.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:15.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:15.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:15.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:15.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-19T20:57:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-19T20:57:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:15.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:15.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:15.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26546' '!=' 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:57:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_21 2026-02-19T20:57:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_21 ']' 2026-02-19T20:57:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-19T20:57:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-19T20:57:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-19T20:57:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:15.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:15.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:15.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2026-02-19T20:57:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:57:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:15.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:15.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:15.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:15.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:57:15.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:15.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:57:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:15.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:15.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:15.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_21 2026-02-19T20:57:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:57:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:57:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:57:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:15.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:57:15.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:57:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-19T20:57:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-19T20:57:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26546' '!=' 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:57:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_21 2026-02-19T20:57:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_21 ']' 2026-02-19T20:57:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-19T20:57:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-19T20:57:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:57:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:57:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:17.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:17.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:17.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_21 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:57:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:57:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-19T20:57:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-19T20:57:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:17.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-19T20:57:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-19T20:57:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:17.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:17.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:17.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:17.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:17.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:17.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:17.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:17.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:17.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:17.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:17.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:17.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26546' '!=' 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:17.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:17.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:57:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_21 2026-02-19T20:57:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_21 ']' 2026-02-19T20:57:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-19T20:57:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-19T20:57:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:19.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:57:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:57:19.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:19.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:19.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:19.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_21 2026-02-19T20:57:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:57:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:57:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:57:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:57:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:19.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:57:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:57:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:57:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:19.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:19.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:19.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-19T20:57:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-19T20:57:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:19.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:19.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:19.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-19T20:57:19.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:19.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:19.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:19.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-19T20:57:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:19.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:19.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:19.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:19.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26546' '!=' 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:57:21.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_21 2026-02-19T20:57:21.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:21.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:21.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:21.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:21.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_21 ']' 2026-02-19T20:57:21.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-19T20:57:21.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-19T20:57:21.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:21.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:21.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:21.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:21.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:21.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:21.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:21.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:21.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:21.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:21.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-19T20:57:21.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:21.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:21.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:21.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:21.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:21.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:21.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:57:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:57:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:57:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:21.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_21 2026-02-19T20:57:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:57:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:57:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:57:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-19T20:57:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-19T20:57:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:21.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:21.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:21.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:21.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:21.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:21.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:21.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:21.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:21.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:21.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:21.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-19T20:57:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-19T20:57:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:21.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:21.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:21.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:21.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26546], [object_number=25, tag_tid=1, entry_tid=26545], [object_number=24, tag_tid=1, entry_tid=26544], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26546' '!=' 'object_number=26, tag_tid=1, entry_tid=26546' 2026-02-19T20:57:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:57:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_21 2026-02-19T20:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_21 ']' 2026-02-19T20:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-19T20:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-19T20:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-19T20:57:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:23.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:23.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:23.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:23.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:23.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:23.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:23.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:23.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2026-02-19T20:57:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2026-02-19T20:57:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:57:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:57:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:57:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:57:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:57:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-19T20:57:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_21 2026-02-19T20:57:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:57:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:57:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:57:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:57:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:57:24.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:24.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:24.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:57:24.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_21 2026-02-19T20:57:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-19T20:57:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-19T20:57:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:24.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:24.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:24.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:24.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:24.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:24.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-19T20:57:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-19T20:57:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T20:57:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-19T20:57:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-19T20:57:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:24.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:24.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-19T20:57:24.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:24.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 6 2026-02-19T20:57:24.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 6 2026-02-19T20:57:24.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:24.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:24.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26546], [object_number=27, tag_tid=1, entry_tid=26543]]], state=connected] 2026-02-19T20:57:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.status 2026-02-19T20:57:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-19T20:57:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2026-02-19T20:57:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:57:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:25.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2026-02-19T20:57:25.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_21': 2026-02-19T20:57:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:57:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:57:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:57:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a644dda991 2026-02-19T20:57:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a644dda991 2026-02-19T20:57:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:57:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:57:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:57:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:57:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:46:36 2026 2026-02-19T20:57:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:46:36 2026 2026-02-19T20:57:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:47:37 2026 2026-02-19T20:57:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:57:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a644dda991 2026-02-19T20:57:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:57:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:57:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 2405118a-2b72-44cd-a98b-afad50e322fd 2026-02-19T20:57:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:57:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_21 snap 2026-02-19T20:57:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T20:57:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:57:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.export 2026-02-19T20:57:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_21.export 2026-02-19T20:57:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_21.export 2026-02-19T20:57:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.export 2026-02-19T20:57:25.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_21.export 2026-02-19T20:57:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:25.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_21.export 2026-02-19T20:57:25.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_21.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_21.export 2026-02-19T20:57:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:57:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_22 2026-02-19T20:57:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_22 snap 2026-02-19T20:57:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:25.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:25.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:25.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:25.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2026-02-19T20:57:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:57:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:57:27.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:57:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2026-02-19T20:57:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:57:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:57:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:57:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:57:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:57:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:57:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2026-02-19T20:57:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:57:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:57:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:57:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:57:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:57:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:57:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:57:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:57:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:57:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2026-02-19T20:57:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-19T20:57:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:57:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_22 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:57:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:57:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_22 2026-02-19T20:57:28.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:57:28.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:57:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:57:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:57:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:57:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:57:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:57:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_22 2026-02-19T20:57:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:57:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:57:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:57:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:57:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_22 2026-02-19T20:57:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:57:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:57:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:57:28.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:28.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:57:28.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:57:28.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:57:28.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:28.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:57:28.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_22 2026-02-19T20:57:28.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:28.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:28.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:28.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_22 ']' 2026-02-19T20:57:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-19T20:57:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-19T20:57:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-19T20:57:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:28.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-19T20:57:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-19T20:57:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-19T20:57:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:28.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:28.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:28.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:28.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:28.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:28.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:28.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-19T20:57:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-19T20:57:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-19T20:57:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-19T20:57:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-19T20:57:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-19T20:57:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:28.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:28.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-19T20:57:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-19T20:57:28.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:28.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36898' '!=' '' 2026-02-19T20:57:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:57:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36898' = '' 2026-02-19T20:57:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:57:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:57:28.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:28.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:57:28.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:57:28.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:28.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:57:28.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:28.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:57:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:57:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36898 2026-02-19T20:57:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 36898 2026-02-19T20:57:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:57:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:28.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:57:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_22 2026-02-19T20:57:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_22 ']' 2026-02-19T20:57:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-19T20:57:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-19T20:57:28.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:28.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-19T20:57:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-19T20:57:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-19T20:57:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-19T20:57:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:28.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:28.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:28.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-19T20:57:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-19T20:57:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-19T20:57:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:28.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-19T20:57:28.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-19T20:57:28.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:28.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-19T20:57:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-19T20:57:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-19T20:57:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-19T20:57:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-19T20:57:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-19T20:57:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:28.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:28.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:28.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:28.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:28.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:28.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:28.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:28.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36898' '!=' 'object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:28.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:28.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:57:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_22 2026-02-19T20:57:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_22 ']' 2026-02-19T20:57:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-19T20:57:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-19T20:57:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:29.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:29.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:29.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:29.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-19T20:57:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-19T20:57:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-19T20:57:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-19T20:57:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-19T20:57:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-19T20:57:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-19T20:57:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-19T20:57:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:29.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:29.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:29.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:29.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:29.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:29.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-19T20:57:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-19T20:57:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-19T20:57:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-19T20:57:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-19T20:57:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:29.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:29.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:29.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:29.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:29.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:29.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36898' '!=' 'object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:29.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:29.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:57:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_22 2026-02-19T20:57:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_22 ']' 2026-02-19T20:57:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-19T20:57:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-19T20:57:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:30.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:30.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:30.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-19T20:57:30.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-19T20:57:30.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:30.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-19T20:57:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:30.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-19T20:57:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-19T20:57:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-19T20:57:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-19T20:57:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-19T20:57:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-19T20:57:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:30.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-19T20:57:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-19T20:57:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-19T20:57:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-19T20:57:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-19T20:57:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36898' '!=' 'object_number=34, tag_tid=1, entry_tid=36898' 2026-02-19T20:57:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:57:32.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_22 2026-02-19T20:57:32.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:32.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:32.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:32.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:32.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_22 ']' 2026-02-19T20:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-19T20:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-19T20:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-19T20:57:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:32.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:32.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:32.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:32.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:32.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:32.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-19T20:57:32.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-19T20:57:32.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:57:32.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:57:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:57:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:57:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:57:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:57:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:57:32.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:32.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:32.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:57:32.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-19T20:57:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-19T20:57:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-19T20:57:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-19T20:57:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-19T20:57:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-19T20:57:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-19T20:57:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-19T20:57:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T20:57:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-19T20:57:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-19T20:57:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-19T20:57:32.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=35, tag_tid=1, entry_tid=36895]]], state=connected] 2026-02-19T20:57:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.status 2026-02-19T20:57:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2026-02-19T20:57:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:57:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:33.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2026-02-19T20:57:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_22': 2026-02-19T20:57:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:57:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11fc7f1314b3 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11fc7f1314b3 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:46:44 2026 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:46:44 2026 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:47:45 2026 2026-02-19T20:57:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:57:33.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11fc7f1314b3 2026-02-19T20:57:33.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:57:33.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:57:33.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: dc9cbe3c-3c30-4147-97dc-5b4bf3a26964 2026-02-19T20:57:33.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:57:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:33.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_22 snap 2026-02-19T20:57:33.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:33.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T20:57:33.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:33.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:57:33.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.export 2026-02-19T20:57:33.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_22.export 2026-02-19T20:57:33.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_22.export 2026-02-19T20:57:33.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.export 2026-02-19T20:57:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_22.export 2026-02-19T20:57:33.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:34.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_22.export 2026-02-19T20:57:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_22.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_22.export 2026-02-19T20:57:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:57:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_23 2026-02-19T20:57:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_23 snap 2026-02-19T20:57:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:34.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:57:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2026-02-19T20:57:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2026-02-19T20:57:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:57:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2026-02-19T20:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2026-02-19T20:57:38.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:57:38.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:57:38.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:38.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:38.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:57:38.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:57:38.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:38.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:57:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:57:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:57:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:57:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:57:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:57:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:57:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-19T20:57:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:57:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:57:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:57:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:57:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:57:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_23 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_23 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:57:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:57:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:57:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:57:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:57:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:57:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:57:39.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_23 2026-02-19T20:57:39.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:39.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:39.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_23 ']' 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-19T20:57:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:39.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-19T20:57:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-19T20:57:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-19T20:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:39.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:39.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-19T20:57:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-19T20:57:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:39.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:39.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:39.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:39.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:39.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:39.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:39.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:39.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35150' '!=' '' 2026-02-19T20:57:39.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:57:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35150' = '' 2026-02-19T20:57:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:57:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:57:39.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:39.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:57:39.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:39.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:57:39.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:39.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:57:39.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:57:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:57:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35150 2026-02-19T20:57:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 35150 2026-02-19T20:57:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:57:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:57:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_23 2026-02-19T20:57:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_23 ']' 2026-02-19T20:57:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-19T20:57:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-19T20:57:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:39.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:39.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:39.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-19T20:57:39.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:39.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:39.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:39.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:39.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:39.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:39.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-19T20:57:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-19T20:57:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-19T20:57:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:39.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-19T20:57:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-19T20:57:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:39.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35150' '!=' 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_23 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_23 ']' 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-19T20:57:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-19T20:57:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-19T20:57:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-19T20:57:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-19T20:57:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-19T20:57:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:40.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:40.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:40.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35150' '!=' 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:40.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:57:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_23 2026-02-19T20:57:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_23 ']' 2026-02-19T20:57:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-19T20:57:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-19T20:57:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-19T20:57:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-19T20:57:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:41.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:41.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:41.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:41.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-19T20:57:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-19T20:57:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:41.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35150' '!=' 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:57:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_23 2026-02-19T20:57:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_23 ']' 2026-02-19T20:57:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-19T20:57:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-19T20:57:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-19T20:57:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-19T20:57:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-19T20:57:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:43.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-19T20:57:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:43.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-19T20:57:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-19T20:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:43.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:43.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:43.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:43.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35150' '!=' 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:57:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_23 2026-02-19T20:57:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_23 ']' 2026-02-19T20:57:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-19T20:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-19T20:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-19T20:57:45.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:45.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-19T20:57:45.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-19T20:57:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-19T20:57:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-19T20:57:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:45.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35150], [object_number=33, tag_tid=1, entry_tid=35149], [object_number=32, tag_tid=1, entry_tid=35148], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:45.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:45.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:45.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35150' '!=' 'object_number=34, tag_tid=1, entry_tid=35150' 2026-02-19T20:57:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:57:47.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_23 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_23 ']' 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:47.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:47.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:47.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:47.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:47.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:47.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:47.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:47.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:47.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:47.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:47.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:47.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:47.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:47.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:47.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-19T20:57:47.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:47.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-19T20:57:47.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:47.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:47.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-19T20:57:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:47.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-19T20:57:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-19T20:57:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T20:57:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-19T20:57:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-19T20:57:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-19T20:57:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:47.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35150], [object_number=35, tag_tid=1, entry_tid=35147]]], state=connected] 2026-02-19T20:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.status 2026-02-19T20:57:47.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:57:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2026-02-19T20:57:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:57:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:48.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2026-02-19T20:57:48.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_23': 2026-02-19T20:57:48.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11fcf38760d9 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11fcf38760d9 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:46:55 2026 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:46:55 2026 2026-02-19T20:57:48.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:47:56 2026 2026-02-19T20:57:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:57:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11fcf38760d9 2026-02-19T20:57:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:57:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:57:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: aae73d36-e56f-4488-bc1c-eda2de4a4a64 2026-02-19T20:57:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:57:48.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:48.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_23 snap 2026-02-19T20:57:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T20:57:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:57:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.export 2026-02-19T20:57:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_23.export 2026-02-19T20:57:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_23.export 2026-02-19T20:57:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.export 2026-02-19T20:57:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_23.export 2026-02-19T20:57:49.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:49.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_23.export 2026-02-19T20:57:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_23.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_23.export 2026-02-19T20:57:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:57:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_24 2026-02-19T20:57:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_24 snap 2026-02-19T20:57:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:57:50.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2026-02-19T20:57:50.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:50.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:50.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:50.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2026-02-19T20:57:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:57:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2026-02-19T20:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2026-02-19T20:57:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:57:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:57:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:57:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:57:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:57:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:57:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:57:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2026-02-19T20:57:52.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:57:52.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:57:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:57:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_24 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_24 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:57:52.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_24 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_24 ']' 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-19T20:57:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23054], [object_number=21, tag_tid=1, entry_tid=23053], [object_number=20, tag_tid=1, entry_tid=23052], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-19T20:57:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-19T20:57:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23054], [object_number=21, tag_tid=1, entry_tid=23053], [object_number=20, tag_tid=1, entry_tid=23052], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:52.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23054' '!=' '' 2026-02-19T20:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23054' = '' 2026-02-19T20:57:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:57:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:57:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:52.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:57:52.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:57:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:57:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:57:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23054 2026-02-19T20:57:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 23054 2026-02-19T20:57:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:57:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:57:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_24 2026-02-19T20:57:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_24 ']' 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:52.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:52.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-19T20:57:52.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:52.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:52.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:52.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:52.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:52.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:52.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:52.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:52.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:52.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:52.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:52.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:52.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:52.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23054], [object_number=21, tag_tid=1, entry_tid=23053], [object_number=20, tag_tid=1, entry_tid=23052], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-19T20:57:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-19T20:57:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:52.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:52.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23054], [object_number=21, tag_tid=1, entry_tid=23053], [object_number=20, tag_tid=1, entry_tid=23052], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:52.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:52.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:52.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23054' '!=' 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:52.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:52.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_24 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_24 ']' 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:53.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:53.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-19T20:57:53.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-19T20:57:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-19T20:57:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-19T20:57:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23054], [object_number=21, tag_tid=1, entry_tid=23053], [object_number=20, tag_tid=1, entry_tid=23052], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:53.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23054], [object_number=21, tag_tid=1, entry_tid=23053], [object_number=20, tag_tid=1, entry_tid=23052], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23054' '!=' 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_24 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_24 ']' 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:54.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:54.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-19T20:57:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23054], [object_number=21, tag_tid=1, entry_tid=23053], [object_number=20, tag_tid=1, entry_tid=23052], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-19T20:57:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-19T20:57:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23054], [object_number=21, tag_tid=1, entry_tid=23053], [object_number=20, tag_tid=1, entry_tid=23052], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:54.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:54.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:54.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23054' '!=' 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:54.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:54.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:57:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_24 2026-02-19T20:57:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_24 ']' 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:56.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:56.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:56.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:56.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-19T20:57:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-19T20:57:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-19T20:57:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23054], [object_number=21, tag_tid=1, entry_tid=23053], [object_number=20, tag_tid=1, entry_tid=23052], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:56.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:56.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-19T20:57:56.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:56.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:56.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:56.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-19T20:57:56.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:56.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:56.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:56.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:56.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:56.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:56.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:56.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:56.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:56.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:56.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:56.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:56.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23054], [object_number=21, tag_tid=1, entry_tid=23053], [object_number=20, tag_tid=1, entry_tid=23052], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23054' '!=' 'object_number=22, tag_tid=1, entry_tid=23054' 2026-02-19T20:57:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:57:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:57:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_24 2026-02-19T20:57:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:57:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_24 ']' 2026-02-19T20:57:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-19T20:57:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-19T20:57:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:57:58.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:58.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:58.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:57:58.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:57:58.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:57:58.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:57:58.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:57:58.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:57:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:57:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-19T20:57:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:57:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:57:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:57:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:57:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:57:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:57:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:57:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:57:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:57:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:57:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:57:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:57:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:57:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:57:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:57:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:57:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:57:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:57:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:57:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:57:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:57:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:57:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:57:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:57:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:57:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:57:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:57:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:57:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-19T20:57:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-19T20:57:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-19T20:57:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:57:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:58.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:58.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:58.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:58.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:58.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:58.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:58.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:58.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-19T20:57:58.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:58.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:58.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-19T20:57:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:57:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:57:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T20:57:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:57:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-19T20:57:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-19T20:57:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:58.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-19T20:57:58.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T20:57:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T20:57:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:57:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23054], [object_number=23, tag_tid=1, entry_tid=23051]]], state=connected] 2026-02-19T20:57:58.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.status 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:57:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:57:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2026-02-19T20:57:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_24': 2026-02-19T20:57:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:57:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:57:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:57:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11fc988f031c 2026-02-19T20:57:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11fc988f031c 2026-02-19T20:57:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:57:59.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:47:04 2026 2026-02-19T20:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:47:04 2026 2026-02-19T20:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:48:05 2026 2026-02-19T20:57:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:57:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11fc988f031c 2026-02-19T20:57:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:57:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:57:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: a95f4bc2-5e3d-4970-9fd8-4a3d1aa1451c 2026-02-19T20:57:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:57:59.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:59.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_24 snap 2026-02-19T20:57:59.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:59.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T20:57:59.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:59.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:57:59.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.export 2026-02-19T20:57:59.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_24.export 2026-02-19T20:57:59.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_24.export 2026-02-19T20:57:59.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.export 2026-02-19T20:57:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_24.export 2026-02-19T20:57:59.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:57:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_24.export 2026-02-19T20:57:59.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_24.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_24.export 2026-02-19T20:57:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:57:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:57:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_25 2026-02-19T20:57:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_25 snap 2026-02-19T20:57:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:57:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:57:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:57:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:57:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2026-02-19T20:58:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:03.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:58:03.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:58:03.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:58:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2026-02-19T20:58:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:58:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:58:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:58:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:58:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:58:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2026-02-19T20:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2026-02-19T20:58:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:03.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_25 2026-02-19T20:58:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:58:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:58:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:58:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:03.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_25 2026-02-19T20:58:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_25 2026-02-19T20:58:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:58:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:58:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:58:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:58:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:04.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:04.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:04.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:04.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_25 2026-02-19T20:58:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:58:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:58:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:58:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:58:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:58:04.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:58:04.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_25 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_25 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:04.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:58:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:58:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_25 2026-02-19T20:58:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_25 ']' 2026-02-19T20:58:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-19T20:58:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-19T20:58:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-19T20:58:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-19T20:58:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:04.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:04.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:04.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:04.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:04.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_25 2026-02-19T20:58:04.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:04.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:04.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-19T20:58:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-19T20:58:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:04.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:58:04.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:58:04.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:04.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:04.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026], [object_number=45, tag_tid=1, entry_tid=47025], [object_number=44, tag_tid=1, entry_tid=47024]]], state=connected] 2026-02-19T20:58:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:58:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-19T20:58:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-19T20:58:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-19T20:58:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-19T20:58:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-19T20:58:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:04.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:58:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:58:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026], [object_number=45, tag_tid=1, entry_tid=47025], [object_number=44, tag_tid=1, entry_tid=47024]]], state=connected] 2026-02-19T20:58:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:04.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=47027' '!=' '' 2026-02-19T20:58:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:58:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=47027' = '' 2026-02-19T20:58:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:58:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:58:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:58:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:58:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:58:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:58:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:58:04.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:04.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:58:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:58:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47027 2026-02-19T20:58:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 47027 2026-02-19T20:58:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:04.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_25 2026-02-19T20:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:04.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_25 ']' 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-19T20:58:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-19T20:58:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:04.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:04.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:04.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:04.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:04.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:04.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:04.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:04.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_25 2026-02-19T20:58:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:04.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-19T20:58:05.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-19T20:58:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-19T20:58:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:58:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:58:05.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:05.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:05.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026], [object_number=45, tag_tid=1, entry_tid=47025], [object_number=44, tag_tid=1, entry_tid=47024]]], state=connected] 2026-02-19T20:58:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:58:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-19T20:58:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-19T20:58:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-19T20:58:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-19T20:58:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:05.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-19T20:58:05.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:58:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:58:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026], [object_number=45, tag_tid=1, entry_tid=47025], [object_number=44, tag_tid=1, entry_tid=47024]]], state=connected] 2026-02-19T20:58:05.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=47027' '!=' 'object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:58:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_25 2026-02-19T20:58:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_25 ']' 2026-02-19T20:58:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-19T20:58:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-19T20:58:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:05.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:05.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:05.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:05.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:05.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:05.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-19T20:58:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:05.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:05.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:05.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_25 2026-02-19T20:58:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:05.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-19T20:58:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-19T20:58:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:05.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:05.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:05.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-19T20:58:05.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-19T20:58:05.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:05.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-19T20:58:05.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:58:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:58:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026], [object_number=45, tag_tid=1, entry_tid=47025], [object_number=44, tag_tid=1, entry_tid=47024]]], state=connected] 2026-02-19T20:58:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:58:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-19T20:58:05.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:05.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:05.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:05.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-19T20:58:05.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:05.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:05.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:05.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:05.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-19T20:58:05.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-19T20:58:05.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-19T20:58:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:58:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:58:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:05.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026], [object_number=45, tag_tid=1, entry_tid=47025], [object_number=44, tag_tid=1, entry_tid=47024]]], state=connected] 2026-02-19T20:58:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=47027' '!=' 'object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_25 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_25 ']' 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:06.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:06.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:06.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-19T20:58:06.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:06.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-19T20:58:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:06.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:06.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:06.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_25 2026-02-19T20:58:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-19T20:58:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-19T20:58:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-19T20:58:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-19T20:58:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:06.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-19T20:58:06.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:58:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:58:06.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:06.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:06.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026], [object_number=45, tag_tid=1, entry_tid=47025], [object_number=44, tag_tid=1, entry_tid=47024]]], state=connected] 2026-02-19T20:58:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:58:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-19T20:58:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-19T20:58:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-19T20:58:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-19T20:58:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:06.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-19T20:58:06.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:06.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:58:06.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:58:06.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:06.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:06.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026], [object_number=45, tag_tid=1, entry_tid=47025], [object_number=44, tag_tid=1, entry_tid=47024]]], state=connected] 2026-02-19T20:58:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=47027' '!=' 'object_number=47, tag_tid=1, entry_tid=47027' 2026-02-19T20:58:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_25 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_25 ']' 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:08.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_25 2026-02-19T20:58:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:08.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:08.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-19T20:58:08.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:08.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_25 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:08.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:08.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:08.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_25 2026-02-19T20:58:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-19T20:58:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-19T20:58:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-19T20:58:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-19T20:58:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:08.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:08.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-19T20:58:08.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:58:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:58:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:08.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:08.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:58:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-19T20:58:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-19T20:58:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T20:58:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-19T20:58:08.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-19T20:58:08.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:08.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-19T20:58:08.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:08.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:58:08.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:58:08.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:08.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:08.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=47027], [object_number=46, tag_tid=1, entry_tid=47026]]], state=connected] 2026-02-19T20:58:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.status 2026-02-19T20:58:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:58:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:58:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2026-02-19T20:58:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:58:08.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:09.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2026-02-19T20:58:09.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_25': 2026-02-19T20:58:09.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a64765e99d 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a64765e99d 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:47:12 2026 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:47:12 2026 2026-02-19T20:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:49:14 2026 2026-02-19T20:58:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:58:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a64765e99d 2026-02-19T20:58:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:58:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:58:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: d8ceb443-5b46-491b-8480-2ed002b9ff2a 2026-02-19T20:58:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:58:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_25 snap 2026-02-19T20:58:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T20:58:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:58:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.export 2026-02-19T20:58:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_25.export 2026-02-19T20:58:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_25.export 2026-02-19T20:58:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.export 2026-02-19T20:58:10.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:58:10.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_25.export 2026-02-19T20:58:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:58:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_25.export 2026-02-19T20:58:10.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_25.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_25.export 2026-02-19T20:58:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:58:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_26 2026-02-19T20:58:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_26 snap 2026-02-19T20:58:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:58:11.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2026-02-19T20:58:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2026-02-19T20:58:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:58:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:11.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:11.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2026-02-19T20:58:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:12.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:58:12.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:58:12.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:58:12.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2026-02-19T20:58:12.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:58:12.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:58:12.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:12.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:12.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:58:12.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:58:12.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:12.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:58:12.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:12.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:58:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2026-02-19T20:58:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:58:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:58:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:58:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:58:12.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2026-02-19T20:58:12.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_26 2026-02-19T20:58:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:58:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:58:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:12.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_26 2026-02-19T20:58:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_26 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:58:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:58:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_26 2026-02-19T20:58:13.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:13.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:13.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_26 2026-02-19T20:58:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:58:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:58:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_26 2026-02-19T20:58:13.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:58:13.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:58:13.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:58:13.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:58:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:58:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:58:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:58:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_26 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_26 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:58:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:58:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_26 2026-02-19T20:58:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_26 ']' 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:13.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:13.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:13.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:13.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:13.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:13.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-19T20:58:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:13.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:13.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:13.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:13.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:13.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:13.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-19T20:58:13.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:13.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:13.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:13.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:13.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:13.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:13.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:13.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:13.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:13.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:13.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_26 2026-02-19T20:58:13.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:13.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:13.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-19T20:58:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-19T20:58:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:13.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-19T20:58:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-19T20:58:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:13.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-19T20:58:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:13.713 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:13.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842], [object_number=37, tag_tid=1, entry_tid=38841], [object_number=36, tag_tid=1, entry_tid=38840]]], state=connected] 2026-02-19T20:58:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-19T20:58:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-19T20:58:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-19T20:58:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-19T20:58:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-19T20:58:13.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842], [object_number=37, tag_tid=1, entry_tid=38841], [object_number=36, tag_tid=1, entry_tid=38840]]], state=connected] 2026-02-19T20:58:13.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38843' '!=' '' 2026-02-19T20:58:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:58:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38843' = '' 2026-02-19T20:58:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:58:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:58:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:58:13.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:13.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:58:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:58:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:58:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:58:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=38843 2026-02-19T20:58:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 38843 2026-02-19T20:58:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_26 2026-02-19T20:58:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_26 ']' 2026-02-19T20:58:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-19T20:58:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-19T20:58:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:13.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:14.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:14.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:14.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:14.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_26 2026-02-19T20:58:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:14.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:14.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:14.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-19T20:58:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-19T20:58:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-19T20:58:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-19T20:58:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:14.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:14.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-19T20:58:14.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842], [object_number=37, tag_tid=1, entry_tid=38841], [object_number=36, tag_tid=1, entry_tid=38840]]], state=connected] 2026-02-19T20:58:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-19T20:58:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-19T20:58:14.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:14.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:14.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:14.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:14.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-19T20:58:14.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-19T20:58:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-19T20:58:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842], [object_number=37, tag_tid=1, entry_tid=38841], [object_number=36, tag_tid=1, entry_tid=38840]]], state=connected] 2026-02-19T20:58:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38843' '!=' 'object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:58:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_26 2026-02-19T20:58:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_26 ']' 2026-02-19T20:58:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-19T20:58:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-19T20:58:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:14.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:14.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:14.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:14.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:14.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-19T20:58:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:14.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:14.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:14.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:14.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:14.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:14.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:14.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:14.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:14.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:14.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:14.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:14.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:14.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:14.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:14.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:14.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:14.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_26 2026-02-19T20:58:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-19T20:58:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:15.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:15.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-19T20:58:15.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:15.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:15.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:15.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:15.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-19T20:58:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-19T20:58:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-19T20:58:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842], [object_number=37, tag_tid=1, entry_tid=38841], [object_number=36, tag_tid=1, entry_tid=38840]]], state=connected] 2026-02-19T20:58:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-19T20:58:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-19T20:58:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-19T20:58:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-19T20:58:15.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-19T20:58:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842], [object_number=37, tag_tid=1, entry_tid=38841], [object_number=36, tag_tid=1, entry_tid=38840]]], state=connected] 2026-02-19T20:58:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38843' '!=' 'object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_26 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_26 ']' 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-19T20:58:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:15.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:15.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:15.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-19T20:58:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:16.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:16.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:16.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_26 2026-02-19T20:58:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-19T20:58:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-19T20:58:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-19T20:58:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-19T20:58:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-19T20:58:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842], [object_number=37, tag_tid=1, entry_tid=38841], [object_number=36, tag_tid=1, entry_tid=38840]]], state=connected] 2026-02-19T20:58:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-19T20:58:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-19T20:58:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-19T20:58:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-19T20:58:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-19T20:58:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842], [object_number=37, tag_tid=1, entry_tid=38841], [object_number=36, tag_tid=1, entry_tid=38840]]], state=connected] 2026-02-19T20:58:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38843' '!=' 'object_number=39, tag_tid=1, entry_tid=38843' 2026-02-19T20:58:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:58:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_26 2026-02-19T20:58:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_26 ']' 2026-02-19T20:58:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-19T20:58:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-19T20:58:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-19T20:58:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-19T20:58:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:18.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:18.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:18.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:18.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:18.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:18.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:18.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:18.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:18.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-19T20:58:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_26 2026-02-19T20:58:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:18.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:18.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:18.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:18.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_26 2026-02-19T20:58:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-19T20:58:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-19T20:58:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:18.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:18.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-19T20:58:18.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-19T20:58:18.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:18.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:18.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-19T20:58:18.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:18.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:18.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-19T20:58:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-19T20:58:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T20:58:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-19T20:58:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-19T20:58:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-19T20:58:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:18.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:18.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:18.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:18.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38843], [object_number=38, tag_tid=1, entry_tid=38842]]], state=connected] 2026-02-19T20:58:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.status 2026-02-19T20:58:18.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:18.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:18.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:18.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2026-02-19T20:58:18.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:18.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:18.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:18.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:18.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:18.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:58:18.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:19.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2026-02-19T20:58:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_26': 2026-02-19T20:58:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:58:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:58:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:58:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 117c1ce76287 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.117c1ce76287 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:47:25 2026 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:47:25 2026 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:48:26 2026 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 117c1ce76287 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: ff691a37-6161-4ec4-bc97-41ec7c7c7153 2026-02-19T20:58:19.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:58:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_26 snap 2026-02-19T20:58:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T20:58:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:58:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.export 2026-02-19T20:58:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_26.export 2026-02-19T20:58:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_26.export 2026-02-19T20:58:19.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.export 2026-02-19T20:58:19.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:58:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_26.export 2026-02-19T20:58:19.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:58:19.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_26.export 2026-02-19T20:58:19.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_26.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_26.export 2026-02-19T20:58:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:58:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_27 2026-02-19T20:58:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_27 snap 2026-02-19T20:58:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:58:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2026-02-19T20:58:21.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:21.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:21.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:21.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2026-02-19T20:58:21.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:21.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:21.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:21.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:58:21.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:21.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:21.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:22.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2026-02-19T20:58:22.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:22.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:22.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:22.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:58:22.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:58:22.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:58:22.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2026-02-19T20:58:22.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:58:22.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:58:22.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:22.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:58:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:58:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:58:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:58:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2026-02-19T20:58:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:58:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:58:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2026-02-19T20:58:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:22.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:22.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:22.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:22.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_27 2026-02-19T20:58:22.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:58:22.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:58:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:58:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_27 2026-02-19T20:58:23.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:23.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:23.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_27 2026-02-19T20:58:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:58:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:58:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:58:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:58:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:58:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:23.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:23.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:23.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:23.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_27 2026-02-19T20:58:23.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:23.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:23.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:23.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_27 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:58:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:58:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_27 2026-02-19T20:58:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:58:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:58:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:58:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:58:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:58:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:58:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:58:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_27 2026-02-19T20:58:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_27 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:23.267 INFO:tasks.ceph.cluster2.osd.0.trial151.stderr:2026-02-19T20:58:23.257+0000 7f785c199640 -1 reset not still connected to 0x557ef623b6c0 2026-02-19T20:58:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_27 2026-02-19T20:58:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_27 ']' 2026-02-19T20:58:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-19T20:58:23.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-19T20:58:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-19T20:58:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:23.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:23.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:23.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-19T20:58:23.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:23.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:23.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:23.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:23.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:23.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:23.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:23.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:23.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:23.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:23.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:23.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:23.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:23.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_27 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-19T20:58:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-19T20:58:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:23.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:58:23.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:58:23.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:23.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:23.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290], [object_number=17, tag_tid=1, entry_tid=18289], [object_number=16, tag_tid=1, entry_tid=18288]]], state=connected] 2026-02-19T20:58:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:23.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:58:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-19T20:58:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-19T20:58:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-19T20:58:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-19T20:58:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-19T20:58:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:58:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:58:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290], [object_number=17, tag_tid=1, entry_tid=18289], [object_number=16, tag_tid=1, entry_tid=18288]]], state=connected] 2026-02-19T20:58:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=18291' '!=' '' 2026-02-19T20:58:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:58:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=18291' = '' 2026-02-19T20:58:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:58:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:58:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:58:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:58:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:58:23.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:58:23.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:58:23.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:58:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:58:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:23.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=18291 2026-02-19T20:58:23.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 18291 2026-02-19T20:58:23.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:23.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:23.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_27 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_27 ']' 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-19T20:58:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:24.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:24.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:24.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:24.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_27 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:24.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:24.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:24.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:24.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-19T20:58:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-19T20:58:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-19T20:58:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-19T20:58:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:24.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:24.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-19T20:58:24.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:58:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:58:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:24.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290], [object_number=17, tag_tid=1, entry_tid=18289], [object_number=16, tag_tid=1, entry_tid=18288]]], state=connected] 2026-02-19T20:58:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-19T20:58:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-19T20:58:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:58:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:58:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:24.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:24.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290], [object_number=17, tag_tid=1, entry_tid=18289], [object_number=16, tag_tid=1, entry_tid=18288]]], state=connected] 2026-02-19T20:58:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=18291' '!=' 'object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:58:24.497 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.476428228088873) > weight (0.25) 2026-02-19T20:58:24.498 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.852290204410507) > weight (0.25) 2026-02-19T20:58:24.498 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9559544384641897) > weight (0.25) 2026-02-19T20:58:24.498 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8947431005401557) > weight (0.25) 2026-02-19T20:58:24.498 INFO:tasks.rbd_mirror_thrash:waiting for 94.41740387002255 secs before thrashing 2026-02-19T20:58:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_27 2026-02-19T20:58:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_27 ']' 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:24.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:24.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_27 2026-02-19T20:58:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-19T20:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-19T20:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-19T20:58:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-19T20:58:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-19T20:58:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:25.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:58:25.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:58:25.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:25.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:25.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290], [object_number=17, tag_tid=1, entry_tid=18289], [object_number=16, tag_tid=1, entry_tid=18288]]], state=connected] 2026-02-19T20:58:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:25.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:58:25.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-19T20:58:25.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:25.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:25.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:25.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-19T20:58:25.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:25.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:25.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:25.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:25.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-19T20:58:25.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-19T20:58:25.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:25.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-19T20:58:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:58:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:58:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290], [object_number=17, tag_tid=1, entry_tid=18289], [object_number=16, tag_tid=1, entry_tid=18288]]], state=connected] 2026-02-19T20:58:25.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=18291' '!=' 'object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:58:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_27 2026-02-19T20:58:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_27 ']' 2026-02-19T20:58:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-19T20:58:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-19T20:58:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:25.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:25.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:25.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:25.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:25.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:25.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-19T20:58:25.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:25.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:25.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:25.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:25.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:25.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:25.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:25.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:25.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:25.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:25.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:25.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:25.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:25.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:25.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:25.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:25.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:25.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:25.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:25.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:25.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:25.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:25.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:25.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:25.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-19T20:58:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:25.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:25.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2026-02-19T20:58:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_27 2026-02-19T20:58:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:26.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:26.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-19T20:58:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-19T20:58:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-19T20:58:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-19T20:58:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-19T20:58:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:58:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:58:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290], [object_number=17, tag_tid=1, entry_tid=18289], [object_number=16, tag_tid=1, entry_tid=18288]]], state=connected] 2026-02-19T20:58:26.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:58:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-19T20:58:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-19T20:58:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-19T20:58:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-19T20:58:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:26.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:26.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-19T20:58:26.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:58:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:58:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290], [object_number=17, tag_tid=1, entry_tid=18289], [object_number=16, tag_tid=1, entry_tid=18288]]], state=connected] 2026-02-19T20:58:26.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=19, tag_tid=1, entry_tid=18291' '!=' 'object_number=19, tag_tid=1, entry_tid=18291' 2026-02-19T20:58:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:58:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_27 2026-02-19T20:58:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_27 ']' 2026-02-19T20:58:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-19T20:58:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-19T20:58:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:27.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:27.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:27.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:27.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:27.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:27.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:27.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:27.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-19T20:58:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:27.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:27.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:27.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:27.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:27.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:28.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:28.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:28.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:28.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:28.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2026-02-19T20:58:28.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:28.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:28.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:28.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:28.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:28.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:28.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:28.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-19T20:58:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_27 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:28.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:28.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:28.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:28.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_27 2026-02-19T20:58:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-19T20:58:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-19T20:58:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-19T20:58:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-19T20:58:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:28.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-19T20:58:28.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:58:28.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:58:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:28.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:58:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-19T20:58:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-19T20:58:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T20:58:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-19T20:58:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-19T20:58:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-19T20:58:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:58:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:58:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18291], [object_number=18, tag_tid=1, entry_tid=18290]]], state=connected] 2026-02-19T20:58:28.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.status 2026-02-19T20:58:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:58:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:58:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2026-02-19T20:58:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:58:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2026-02-19T20:58:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_27': 2026-02-19T20:58:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:58:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:58:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:58:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11a2f4c91222 2026-02-19T20:58:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11a2f4c91222 2026-02-19T20:58:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:58:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:58:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:58:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:58:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:47:37 2026 2026-02-19T20:58:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:47:37 2026 2026-02-19T20:58:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:47:37 2026 2026-02-19T20:58:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:58:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11a2f4c91222 2026-02-19T20:58:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:58:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:58:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: bc93a362-0075-4b5e-8c78-6be5d453a095 2026-02-19T20:58:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:58:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_27 snap 2026-02-19T20:58:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T20:58:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:58:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.export 2026-02-19T20:58:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_27.export 2026-02-19T20:58:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_27.export 2026-02-19T20:58:29.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.export 2026-02-19T20:58:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:58:29.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_27.export 2026-02-19T20:58:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:58:29.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_27.export 2026-02-19T20:58:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_27.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_27.export 2026-02-19T20:58:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:58:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_28 2026-02-19T20:58:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_28 snap 2026-02-19T20:58:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:58:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:29.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:58:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2026-02-19T20:58:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:58:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2026-02-19T20:58:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:58:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:58:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:31.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2026-02-19T20:58:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:58:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:58:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:58:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:58:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2026-02-19T20:58:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:58:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:58:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:58:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:58:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:58:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:58:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-19T20:58:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:58:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:58:31.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:58:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-19T20:58:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-19T20:58:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-02-19T20:58:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:58:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:58:31.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-02-19T20:58:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-19T20:58:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2026-02-19T20:58:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:58:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:58:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:58:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2026-02-19T20:58:32.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-19T20:58:32.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:32.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_28 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:58:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:58:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_28 2026-02-19T20:58:32.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:58:32.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:58:32.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:58:32.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:58:32.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:58:32.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:58:32.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:58:32.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:32.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_28 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_28 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:58:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:58:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:58:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_28 2026-02-19T20:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_28 ']' 2026-02-19T20:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-19T20:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-19T20:58:32.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:32.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:32.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:32.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:32.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:32.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:32.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:32.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:32.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:32.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:32.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-19T20:58:32.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:32.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:32.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:32.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:32.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:32.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:32.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:32.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-19T20:58:32.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-19T20:58:32.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:32.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:32.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:32.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:32.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-19T20:58:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-19T20:58:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:32.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:32.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-19T20:58:32.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-19T20:58:32.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:32.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_28 2026-02-19T20:58:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:32.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:32.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:32.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:32.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:32.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_28 2026-02-19T20:58:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-19T20:58:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_28 2026-02-19T20:58:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-19T20:58:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_28 2026-02-19T20:58:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-19T20:58:32.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-19T20:58:32.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:32.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.status 2026-02-19T20:58:32.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-19T20:58:32.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.status 2026-02-19T20:58:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32399], [object_number=30, tag_tid=1, entry_tid=32398]]], state=connected] 2026-02-19T20:58:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32399], [object_number=30, tag_tid=1, entry_tid=32398]]], state=connected] 2026-02-19T20:58:32.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.status 2026-02-19T20:58:32.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-19T20:58:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_28 2026-02-19T20:58:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-19T20:58:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_28 2026-02-19T20:58:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-19T20:58:32.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-19T20:58:32.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:32.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.status 2026-02-19T20:58:32.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-19T20:58:32.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.status 2026-02-19T20:58:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32399], [object_number=30, tag_tid=1, entry_tid=32398]]], state=connected] 2026-02-19T20:58:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=32399], [object_number=30, tag_tid=1, entry_tid=32398]]], state=connected] 2026-02-19T20:58:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.status 2026-02-19T20:58:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2026-02-19T20:58:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:58:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:58:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:33.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2026-02-19T20:58:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_28': 2026-02-19T20:58:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:58:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:58:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:58:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11fc4ba904bf 2026-02-19T20:58:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11fc4ba904bf 2026-02-19T20:58:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:47:42 2026 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:47:42 2026 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:48:43 2026 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11fc4ba904bf 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:58:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 6209872a-082c-4884-ace7-de5af28ee564 2026-02-19T20:58:33.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:58:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_28 snap 2026-02-19T20:58:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T20:58:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:58:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.export 2026-02-19T20:58:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_28.export 2026-02-19T20:58:33.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_28.export 2026-02-19T20:58:33.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.export 2026-02-19T20:58:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:58:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_28.export 2026-02-19T20:58:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:58:34.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_28.export 2026-02-19T20:58:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_28.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_28.export 2026-02-19T20:58:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:58:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_29 2026-02-19T20:58:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_29 snap 2026-02-19T20:58:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:34.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:58:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2026-02-19T20:58:36.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2026-02-19T20:58:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:58:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2026-02-19T20:58:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:58:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:58:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:58:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:58:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2026-02-19T20:58:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:58:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:58:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2026-02-19T20:58:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_29 2026-02-19T20:58:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:58:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:37.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:37.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:37.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:58:37.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:58:37.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_29 2026-02-19T20:58:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_29 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:58:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:58:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:38.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:38.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:38.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:38.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_29 2026-02-19T20:58:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_29 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:58:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:58:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:58:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:38.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_29 2026-02-19T20:58:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:58:38.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:58:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:58:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:58:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:58:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:58:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:58:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_29 2026-02-19T20:58:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:58:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_29 2026-02-19T20:58:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:58:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:58:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_29 2026-02-19T20:58:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:38.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_29 ']' 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:38.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:38.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:38.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:38.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-19T20:58:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:38.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:38.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:38.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:38.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:38.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:38.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:38.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:38.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:38.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-19T20:58:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:38.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:38.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:38.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:38.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:38.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:38.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-19T20:58:38.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:38.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:38.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:38.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:38.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:38.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:38.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:38.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_29 2026-02-19T20:58:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:38.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:38.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:38.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:38.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-19T20:58:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-19T20:58:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-19T20:58:38.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-19T20:58:38.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:38.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:38.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-19T20:58:38.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162], [object_number=37, tag_tid=1, entry_tid=40161], [object_number=36, tag_tid=1, entry_tid=40160]]], state=connected] 2026-02-19T20:58:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-19T20:58:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-19T20:58:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-19T20:58:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-19T20:58:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-19T20:58:38.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:38.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:38.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162], [object_number=37, tag_tid=1, entry_tid=40161], [object_number=36, tag_tid=1, entry_tid=40160]]], state=connected] 2026-02-19T20:58:38.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=40163' '!=' '' 2026-02-19T20:58:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:58:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=40163' = '' 2026-02-19T20:58:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:38.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:38.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:58:38.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:58:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:58:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:58:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:58:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:38.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:58:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:58:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40163 2026-02-19T20:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 40163 2026-02-19T20:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_29 2026-02-19T20:58:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_29 ']' 2026-02-19T20:58:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-19T20:58:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-19T20:58:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-19T20:58:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:39.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:39.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:39.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:39.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:39.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:39.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:39.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:39.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_29 2026-02-19T20:58:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:39.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-19T20:58:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-19T20:58:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-19T20:58:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-19T20:58:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-19T20:58:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:39.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:39.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:39.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:39.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:39.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162], [object_number=37, tag_tid=1, entry_tid=40161], [object_number=36, tag_tid=1, entry_tid=40160]]], state=connected] 2026-02-19T20:58:39.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:39.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-19T20:58:39.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:39.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:39.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-19T20:58:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-19T20:58:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-19T20:58:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-19T20:58:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162], [object_number=37, tag_tid=1, entry_tid=40161], [object_number=36, tag_tid=1, entry_tid=40160]]], state=connected] 2026-02-19T20:58:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=40163' '!=' 'object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:58:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_29 2026-02-19T20:58:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_29 ']' 2026-02-19T20:58:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-19T20:58:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-19T20:58:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-19T20:58:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-19T20:58:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-19T20:58:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:39.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:39.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:39.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:39.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:39.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:39.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:39.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:39.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_29 2026-02-19T20:58:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-19T20:58:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-19T20:58:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-19T20:58:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-19T20:58:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-19T20:58:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162], [object_number=37, tag_tid=1, entry_tid=40161], [object_number=36, tag_tid=1, entry_tid=40160]]], state=connected] 2026-02-19T20:58:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-19T20:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-19T20:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-19T20:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-19T20:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-19T20:58:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:40.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:40.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:40.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:40.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:40.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162], [object_number=37, tag_tid=1, entry_tid=40161], [object_number=36, tag_tid=1, entry_tid=40160]]], state=connected] 2026-02-19T20:58:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=40163' '!=' 'object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:58:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_29 2026-02-19T20:58:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_29 ']' 2026-02-19T20:58:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-19T20:58:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-19T20:58:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-19T20:58:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-19T20:58:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:41.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:41.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-19T20:58:41.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:41.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:41.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:41.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_29 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-19T20:58:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-19T20:58:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-19T20:58:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-19T20:58:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-19T20:58:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:41.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162], [object_number=37, tag_tid=1, entry_tid=40161], [object_number=36, tag_tid=1, entry_tid=40160]]], state=connected] 2026-02-19T20:58:41.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-19T20:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-19T20:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-19T20:58:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-19T20:58:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-19T20:58:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162], [object_number=37, tag_tid=1, entry_tid=40161], [object_number=36, tag_tid=1, entry_tid=40160]]], state=connected] 2026-02-19T20:58:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=40163' '!=' 'object_number=39, tag_tid=1, entry_tid=40163' 2026-02-19T20:58:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:58:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_29 2026-02-19T20:58:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_29 ']' 2026-02-19T20:58:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-19T20:58:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-19T20:58:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-19T20:58:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-19T20:58:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:43.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-19T20:58:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:43.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_29 2026-02-19T20:58:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:43.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:43.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:43.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:43.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_29 2026-02-19T20:58:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-19T20:58:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-19T20:58:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-19T20:58:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-19T20:58:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-19T20:58:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:43.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:43.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:43.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:43.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:43.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:43.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:43.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-19T20:58:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-19T20:58:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T20:58:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-19T20:58:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-19T20:58:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-19T20:58:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:58:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:58:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40163], [object_number=38, tag_tid=1, entry_tid=40162]]], state=connected] 2026-02-19T20:58:43.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.status 2026-02-19T20:58:43.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:43.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:58:43.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:43.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2026-02-19T20:58:43.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:43.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:43.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:43.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:43.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:43.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:58:43.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:44.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2026-02-19T20:58:44.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_29': 2026-02-19T20:58:44.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:58:44.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:58:44.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:58:44.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a662efdd35 2026-02-19T20:58:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a662efdd35 2026-02-19T20:58:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:58:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:58:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:58:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:58:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:47:51 2026 2026-02-19T20:58:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:47:51 2026 2026-02-19T20:58:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:48:52 2026 2026-02-19T20:58:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:58:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a662efdd35 2026-02-19T20:58:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:58:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:58:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: c738fa13-194f-46f1-a4a1-8abf31b95464 2026-02-19T20:58:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:58:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_29 snap 2026-02-19T20:58:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T20:58:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:58:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.export 2026-02-19T20:58:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_29.export 2026-02-19T20:58:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_29.export 2026-02-19T20:58:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.export 2026-02-19T20:58:44.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:58:44.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_29.export 2026-02-19T20:58:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:58:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_29.export 2026-02-19T20:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_29.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_29.export 2026-02-19T20:58:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:58:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_30 2026-02-19T20:58:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_30 snap 2026-02-19T20:58:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:58:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2026-02-19T20:58:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2026-02-19T20:58:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:58:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:58:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2026-02-19T20:58:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:58:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:58:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:58:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2026-02-19T20:58:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:58:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:58:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:58:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:58:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:58:48.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:48.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:58:48.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:48.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-19T20:58:48.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:58:48.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:48.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:48.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:48.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:58:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:48.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:48.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-19T20:58:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:48.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:48.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:48.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:58:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:58:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-19T20:58:49.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:49.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:49.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-19T20:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:58:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:58:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:58:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-19T20:58:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:58:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-02-19T20:58:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:58:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:58:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:58:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:58:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:58:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:58:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:58:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-02-19T20:58:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:58:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:58:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:58:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:58:49.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:58:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:58:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:58:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_30 2026-02-19T20:58:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_30 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:49.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_30 2026-02-19T20:58:49.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:49.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:49.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:49.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_30 ']' 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:49.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-19T20:58:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:49.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:49.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:49.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:49.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:49.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:49.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-19T20:58:49.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:49.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:49.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:49.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:49.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:49.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:49.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:49.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:49.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:49.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:49.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-19T20:58:49.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:49.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:49.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:49.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:49.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:49.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:49.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:49.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:49.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:49.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-19T20:58:49.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:49.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:49.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:49.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-19T20:58:49.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:49.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:49.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:49.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:49.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:49.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:49.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:49.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:49.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:49.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-19T20:58:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-19T20:58:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:49.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:49.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:49.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:49.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:49.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:49.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31462' '!=' '' 2026-02-19T20:58:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:58:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31462' = '' 2026-02-19T20:58:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:49.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:49.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:58:49.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:49.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:58:49.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:49.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:58:49.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:49.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:58:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:58:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:58:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:58:49.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:58:49.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=31462 2026-02-19T20:58:49.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 31462 2026-02-19T20:58:49.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:58:49.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:49.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:58:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_30 2026-02-19T20:58:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:49.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:49.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:49.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:49.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_30 ']' 2026-02-19T20:58:49.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-19T20:58:49.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-19T20:58:49.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:49.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:49.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:49.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-19T20:58:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-19T20:58:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:50.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:50.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:50.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:50.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:50.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:50.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:50.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-19T20:58:50.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:50.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:50.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:50.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:50.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:50.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:50.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:50.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:50.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-19T20:58:50.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:50.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:50.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:50.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:50.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-19T20:58:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-19T20:58:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:50.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:50.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:50.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:50.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:50.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-19T20:58:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-19T20:58:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:50.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:50.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:50.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:50.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:50.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:50.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:50.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31462' '!=' 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:58:50.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_30 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_30 ']' 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-19T20:58:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:50.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-19T20:58:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:50.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:50.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:50.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:50.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:50.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:50.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:50.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:50.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:50.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:50.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:50.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:50.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:50.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:50.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-19T20:58:50.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-19T20:58:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-19T20:58:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:51.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:51.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:51.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:51.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:51.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:51.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:51.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:51.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:51.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-19T20:58:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-19T20:58:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31462' '!=' 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_30 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_30 ']' 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:51.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:52.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:52.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:52.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-19T20:58:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-19T20:58:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:52.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:52.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:52.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-19T20:58:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-19T20:58:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:52.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:52.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:52.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:52.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:52.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:52.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:52.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:52.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-19T20:58:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-19T20:58:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:52.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31462' '!=' 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:58:53.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_30 2026-02-19T20:58:53.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:53.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:53.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:53.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:53.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_30 ']' 2026-02-19T20:58:53.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-19T20:58:53.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-19T20:58:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:53.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:53.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:53.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:53.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:53.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:53.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:53.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:53.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:53.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:53.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:53.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:53.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:53.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:53.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-19T20:58:53.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:53.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:54.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:54.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:54.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:54.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:54.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-19T20:58:54.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:54.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:54.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:54.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:54.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:54.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:54.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:54.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:54.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:54.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:54.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:54.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:54.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-19T20:58:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:54.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:54.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:54.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-19T20:58:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-19T20:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:54.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:54.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:54.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:54.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:54.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-19T20:58:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-19T20:58:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:54.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:54.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:54.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:54.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31462' '!=' 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:54.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:54.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:58:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_30 2026-02-19T20:58:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_30 ']' 2026-02-19T20:58:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-19T20:58:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-19T20:58:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:56.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:56.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:56.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:56.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:56.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-19T20:58:56.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:56.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:56.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:56.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:56.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:56.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:56.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:56.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:56.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:56.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:56.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:56.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:56.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:56.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-19T20:58:56.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:56.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:56.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:56.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:56.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:56.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:56.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:56.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-19T20:58:56.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:56.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-19T20:58:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:56.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-19T20:58:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-19T20:58:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:56.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:56.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:56.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:56.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:56.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:56.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:56.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:56.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-19T20:58:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-19T20:58:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:56.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:56.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:56.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:56.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:56.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:56.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:56.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:56.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:56.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31462], [object_number=29, tag_tid=1, entry_tid=31461], [object_number=28, tag_tid=1, entry_tid=31460], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31462' '!=' 'object_number=30, tag_tid=1, entry_tid=31462' 2026-02-19T20:58:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:58:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:58:58.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_30 2026-02-19T20:58:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:58:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_30 ']' 2026-02-19T20:58:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-19T20:58:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-19T20:58:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:58:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:58:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:58:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:58:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:58:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:58:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:58:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-19T20:58:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:58:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:58:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:58:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:58:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:58:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-19T20:58:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:58:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:58:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:58:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:58:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:58:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:58.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:58:58.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:58:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:58:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:58:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:58:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:58:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:58:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-19T20:58:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:58:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:58:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:58:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-19T20:58:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:58:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:58:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:58:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:58:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:58:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:58:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:58:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:58:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:58:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:58:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:58:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:58:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:58:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:58:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:58:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:58:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:58:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:58:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:58:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:58:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:58:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:58:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:58:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:58:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:58:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:58:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:58:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:58:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-19T20:58:59.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-19T20:58:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-19T20:58:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:58:59.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:59.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:59.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:59.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:59.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:59.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-19T20:58:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-19T20:58:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:58:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:58:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T20:58:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:58:59.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-19T20:58:59.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-19T20:58:59.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:58:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-19T20:58:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 7 2026-02-19T20:58:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 7 2026-02-19T20:58:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:58:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31462], [object_number=31, tag_tid=1, entry_tid=31459]]], state=connected] 2026-02-19T20:58:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.status 2026-02-19T20:58:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:59.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-19T20:58:59.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:58:59.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2026-02-19T20:58:59.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:58:59.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:58:59.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:58:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:58:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:58:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:58:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2026-02-19T20:59:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_30': 2026-02-19T20:59:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:59:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:59:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:59:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 117cc652973f 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.117cc652973f 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:48:04 2026 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:48:04 2026 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:49:05 2026 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 117cc652973f 2026-02-19T20:59:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:59:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:59:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 4aa9fa1b-dec5-4577-9825-3b3e2bb68c28 2026-02-19T20:59:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:59:00.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:00.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_30 snap 2026-02-19T20:59:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T20:59:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:59:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.export 2026-02-19T20:59:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_30.export 2026-02-19T20:59:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_30.export 2026-02-19T20:59:00.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.export 2026-02-19T20:59:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_30.export 2026-02-19T20:59:00.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_30.export 2026-02-19T20:59:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_30.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_30.export 2026-02-19T20:59:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:59:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_31 2026-02-19T20:59:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_31 snap 2026-02-19T20:59:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:59:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2026-02-19T20:59:01.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:01.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:01.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:01.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2026-02-19T20:59:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:59:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:02.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2026-02-19T20:59:02.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:02.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:02.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:02.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:59:02.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:59:02.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:59:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2026-02-19T20:59:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:59:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:59:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:59:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:59:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:59:02.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:02.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:59:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:59:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2026-02-19T20:59:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-19T20:59:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:59:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31 2026-02-19T20:59:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:59:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:59:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31 2026-02-19T20:59:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:59:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:59:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:59:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:59:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:59:03.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:59:03.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:59:03.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:03.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_31 2026-02-19T20:59:03.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:03.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:03.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_31 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:59:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:59:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_31 2026-02-19T20:59:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_31 ']' 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:03.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:03.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:03.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:03.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:03.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-19T20:59:03.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:03.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:03.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:03.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:03.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:03.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:03.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-19T20:59:03.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-19T20:59:03.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:03.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:03.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:03.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-19T20:59:03.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:03.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:03.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-19T20:59:03.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:03.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-19T20:59:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-19T20:59:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-19T20:59:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-19T20:59:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:03.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-19T20:59:03.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:59:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:59:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:03.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19264], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262], [object_number=17, tag_tid=1, entry_tid=19261]]], state=connected] 2026-02-19T20:59:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:59:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-19T20:59:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-19T20:59:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:03.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-19T20:59:03.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-19T20:59:03.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:03.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-19T20:59:03.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:59:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:59:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:03.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19264], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262], [object_number=17, tag_tid=1, entry_tid=19261]]], state=connected] 2026-02-19T20:59:03.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19264' '!=' '' 2026-02-19T20:59:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:59:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19264' = '' 2026-02-19T20:59:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:03.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:59:03.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:59:03.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:03.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:59:03.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:03.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:59:03.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:59:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:59:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:59:03.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:03.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:59:03.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:03.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:59:03.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:03.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=19264 2026-02-19T20:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 19264 2026-02-19T20:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_31 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_31 ']' 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:03.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:03.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:03.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:03.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-19T20:59:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-19T20:59:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-19T20:59:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-19T20:59:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-19T20:59:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:59:03.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:59:03.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:03.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:03.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19264], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262], [object_number=17, tag_tid=1, entry_tid=19261]]], state=connected] 2026-02-19T20:59:03.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:59:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-19T20:59:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-19T20:59:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-19T20:59:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-19T20:59:03.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-19T20:59:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:59:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:59:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19264], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262], [object_number=17, tag_tid=1, entry_tid=19261]]], state=connected] 2026-02-19T20:59:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19264' '!=' 'object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:03.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:03.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:59:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_31 2026-02-19T20:59:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_31 ']' 2026-02-19T20:59:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-19T20:59:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-19T20:59:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-19T20:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-19T20:59:04.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-19T20:59:04.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:04.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-19T20:59:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-19T20:59:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-19T20:59:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-19T20:59:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-19T20:59:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-19T20:59:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-19T20:59:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:59:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:59:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:04.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19264], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262], [object_number=17, tag_tid=1, entry_tid=19261]]], state=connected] 2026-02-19T20:59:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:59:04.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-19T20:59:04.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:04.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:04.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:04.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-19T20:59:04.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:04.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:04.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:04.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:04.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-19T20:59:04.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-19T20:59:04.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:04.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:04.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-19T20:59:04.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:04.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:59:04.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:59:04.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:04.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:04.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19264], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262], [object_number=17, tag_tid=1, entry_tid=19261]]], state=connected] 2026-02-19T20:59:04.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:04.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:04.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:04.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19264' '!=' 'object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:04.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:04.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:59:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_31 2026-02-19T20:59:05.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:05.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:05.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:05.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:05.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_31 ']' 2026-02-19T20:59:05.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-19T20:59:05.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-19T20:59:05.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:05.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:05.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:05.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:05.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:05.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:05.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:05.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:05.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:05.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:05.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-19T20:59:05.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:05.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:05.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:05.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:05.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-19T20:59:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-19T20:59:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-19T20:59:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-19T20:59:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-19T20:59:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-19T20:59:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:05.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-19T20:59:05.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-19T20:59:05.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:05.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:05.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-19T20:59:05.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:59:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:59:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19264], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262], [object_number=17, tag_tid=1, entry_tid=19261]]], state=connected] 2026-02-19T20:59:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:59:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-19T20:59:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-19T20:59:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-19T20:59:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-19T20:59:05.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:05.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:05.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-19T20:59:05.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:06.002 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.2722796280423876) > weight (0.25) 2026-02-19T20:59:06.002 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7266411944736028) > weight (0.25) 2026-02-19T20:59:06.002 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-19T20:59:06.003 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-19T20:59:06.003 INFO:tasks.rbd_mirror_thrash:waiting for 1.952910235550872 secs before reviving daemons 2026-02-19T20:59:06.003 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T20:59:05.993+0000 7fb12de7f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 82727) UID: 1000 2026-02-19T20:59:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:59:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:59:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=19264], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262], [object_number=17, tag_tid=1, entry_tid=19261]]], state=connected] 2026-02-19T20:59:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=19264' '!=' 'object_number=16, tag_tid=1, entry_tid=19264' 2026-02-19T20:59:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:59:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_31 2026-02-19T20:59:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:07.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:07.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:07.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_31 ']' 2026-02-19T20:59:07.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-19T20:59:07.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-19T20:59:07.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:07.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-19T20:59:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:07.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:07.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:07.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:07.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:07.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-19T20:59:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-19T20:59:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-19T20:59:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:07.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-19T20:59:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-19T20:59:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-19T20:59:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-19T20:59:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-19T20:59:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-19T20:59:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:59:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:59:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:59:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-19T20:59:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-19T20:59:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T20:59:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-19T20:59:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-19T20:59:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-19T20:59:07.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T20:59:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T20:59:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19263], [object_number=18, tag_tid=1, entry_tid=19262]]], state=connected] 2026-02-19T20:59:07.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.status 2026-02-19T20:59:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:59:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T20:59:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2026-02-19T20:59:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:59:07.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:07.957 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-19T20:59:07.957 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T20:59:07.957 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-19T20:59:07.957 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-19T20:59:07.958 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-19T20:59:07.960 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-19T20:59:07.960 INFO:tasks.rbd_mirror_thrash:waiting for 73.7281464952246 secs before thrashing 2026-02-19T20:59:08.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2026-02-19T20:59:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_31': 2026-02-19T20:59:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:59:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:59:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:59:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11a2d7d5020 2026-02-19T20:59:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11a2d7d5020 2026-02-19T20:59:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:48:13 2026 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:48:13 2026 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:48:13 2026 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11a2d7d5020 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 96b77a01-802d-450a-965e-b51abc0bfe7b 2026-02-19T20:59:08.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:59:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_31 snap 2026-02-19T20:59:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T20:59:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:59:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.export 2026-02-19T20:59:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_31.export 2026-02-19T20:59:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_31.export 2026-02-19T20:59:08.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.export 2026-02-19T20:59:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_31.export 2026-02-19T20:59:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_31.export 2026-02-19T20:59:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_31.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_31.export 2026-02-19T20:59:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:59:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_32 2026-02-19T20:59:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_32 snap 2026-02-19T20:59:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:59:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2026-02-19T20:59:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2026-02-19T20:59:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:59:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:12.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2026-02-19T20:59:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:59:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:59:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:59:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2026-02-19T20:59:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:59:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:59:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:59:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:59:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:59:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:59:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2026-02-19T20:59:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:59:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:59:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:59:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2026-02-19T20:59:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-19T20:59:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:59:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2026-02-19T20:59:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:59:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:59:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:59:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:13.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:13.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:13.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:13.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2026-02-19T20:59:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:59:13.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:59:13.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:59:13.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:59:13.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:59:13.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:59:13.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:59:13.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:13.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_32 2026-02-19T20:59:13.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:13.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:13.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:13.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:13.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:13.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:59:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_32 2026-02-19T20:59:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:59:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:59:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:13.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:13.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:13.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_32 2026-02-19T20:59:13.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_32 ']' 2026-02-19T20:59:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-19T20:59:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-19T20:59:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-19T20:59:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-19T20:59:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-19T20:59:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-19T20:59:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:13.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:13.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:13.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:13.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-19T20:59:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-19T20:59:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-19T20:59:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-19T20:59:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-19T20:59:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-19T20:59:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:59:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:59:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494], [object_number=45, tag_tid=1, entry_tid=46493], [object_number=44, tag_tid=1, entry_tid=46492]]], state=connected] 2026-02-19T20:59:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:59:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-19T20:59:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:13.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-19T20:59:13.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:13.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:13.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:13.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:13.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-19T20:59:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-19T20:59:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-19T20:59:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:59:13.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:59:13.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:13.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:13.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494], [object_number=45, tag_tid=1, entry_tid=46493], [object_number=44, tag_tid=1, entry_tid=46492]]], state=connected] 2026-02-19T20:59:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=46495' '!=' '' 2026-02-19T20:59:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:59:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=46495' = '' 2026-02-19T20:59:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:59:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:59:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:59:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:59:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:59:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:59:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:59:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:59:13.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:13.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:59:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46495 2026-02-19T20:59:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 46495 2026-02-19T20:59:13.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:13.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_32 2026-02-19T20:59:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_32 ']' 2026-02-19T20:59:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-19T20:59:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-19T20:59:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-19T20:59:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:13.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:13.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:13.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:13.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:13.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:13.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:13.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:13.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-19T20:59:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-19T20:59:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:13.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:13.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-19T20:59:13.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:13.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:13.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:13.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:13.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:13.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:13.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-19T20:59:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-19T20:59:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:13.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-19T20:59:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:13.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-19T20:59:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-19T20:59:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:13.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-19T20:59:13.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:59:13.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:59:13.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:13.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:13.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494], [object_number=45, tag_tid=1, entry_tid=46493], [object_number=44, tag_tid=1, entry_tid=46492]]], state=connected] 2026-02-19T20:59:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-19T20:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-19T20:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-19T20:59:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-19T20:59:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-19T20:59:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:59:13.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:59:13.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:13.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:13.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494], [object_number=45, tag_tid=1, entry_tid=46493], [object_number=44, tag_tid=1, entry_tid=46492]]], state=connected] 2026-02-19T20:59:13.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=46495' '!=' 'object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:13.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:59:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_32 2026-02-19T20:59:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_32 ']' 2026-02-19T20:59:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-19T20:59:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-19T20:59:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-19T20:59:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-19T20:59:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-19T20:59:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-19T20:59:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:14.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:14.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:14.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:14.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:14.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-19T20:59:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-19T20:59:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-19T20:59:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-19T20:59:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-19T20:59:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-19T20:59:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:14.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:59:14.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:59:14.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:14.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:14.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494], [object_number=45, tag_tid=1, entry_tid=46493], [object_number=44, tag_tid=1, entry_tid=46492]]], state=connected] 2026-02-19T20:59:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:59:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-19T20:59:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-19T20:59:14.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:14.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:14.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:14.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:14.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-19T20:59:14.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-19T20:59:14.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-19T20:59:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:59:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:59:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494], [object_number=45, tag_tid=1, entry_tid=46493], [object_number=44, tag_tid=1, entry_tid=46492]]], state=connected] 2026-02-19T20:59:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=46495' '!=' 'object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_32 2026-02-19T20:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_32 ']' 2026-02-19T20:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-19T20:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-19T20:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-19T20:59:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-19T20:59:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-19T20:59:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:15.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:15.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-19T20:59:15.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:15.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:15.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:15.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-19T20:59:15.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-19T20:59:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-19T20:59:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-19T20:59:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-19T20:59:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-19T20:59:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:59:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:59:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494], [object_number=45, tag_tid=1, entry_tid=46493], [object_number=44, tag_tid=1, entry_tid=46492]]], state=connected] 2026-02-19T20:59:15.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:15.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:59:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-19T20:59:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-19T20:59:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-19T20:59:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-19T20:59:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-19T20:59:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494], [object_number=45, tag_tid=1, entry_tid=46493], [object_number=44, tag_tid=1, entry_tid=46492]]], state=connected] 2026-02-19T20:59:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=46495' '!=' 'object_number=47, tag_tid=1, entry_tid=46495' 2026-02-19T20:59:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:15.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:59:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_32 2026-02-19T20:59:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_32 ']' 2026-02-19T20:59:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-19T20:59:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-19T20:59:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:17.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:17.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:17.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:17.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:17.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-19T20:59:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-19T20:59:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:17.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-19T20:59:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:17.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-19T20:59:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-19T20:59:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:17.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-19T20:59:17.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:59:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:59:17.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:17.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:17.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:59:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-19T20:59:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-19T20:59:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T20:59:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-19T20:59:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-19T20:59:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-19T20:59:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T20:59:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T20:59:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:17.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46495], [object_number=46, tag_tid=1, entry_tid=46494]]], state=connected] 2026-02-19T20:59:17.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.status 2026-02-19T20:59:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:59:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T20:59:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2026-02-19T20:59:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:59:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:18.483 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2026-02-19T20:59:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_32': 2026-02-19T20:59:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:59:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11fcc0706737 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11fcc0706737 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:48:24 2026 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:48:24 2026 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:49:25 2026 2026-02-19T20:59:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:59:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11fcc0706737 2026-02-19T20:59:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:59:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:59:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 3a18f5f4-b23c-4fbc-bbec-0dff64ec1f7b 2026-02-19T20:59:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_32 snap 2026-02-19T20:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T20:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.export 2026-02-19T20:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_32.export 2026-02-19T20:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_32.export 2026-02-19T20:59:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.export 2026-02-19T20:59:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:18.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_32.export 2026-02-19T20:59:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:18.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_32.export 2026-02-19T20:59:18.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_32.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_32.export 2026-02-19T20:59:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:19.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:59:19.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_33 2026-02-19T20:59:19.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_33 snap 2026-02-19T20:59:19.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:19.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:19.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:19.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:19.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2026-02-19T20:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2026-02-19T20:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:20.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:20.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:21.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2026-02-19T20:59:21.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:21.875 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:59:21.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:59:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:59:21.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:21.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:59:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:59:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:59:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:21.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:21.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2026-02-19T20:59:21.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:59:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:21.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:59:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:59:21.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:21.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:21.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:21.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2026-02-19T20:59:22.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:22.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:59:22.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2026-02-19T20:59:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:59:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:59:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:22.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:22.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2026-02-19T20:59:22.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:59:22.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:59:22.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:59:22.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:59:22.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:59:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:59:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:59:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_33 2026-02-19T20:59:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:59:22.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_33 2026-02-19T20:59:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:59:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:59:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_33 2026-02-19T20:59:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_33 ']' 2026-02-19T20:59:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-19T20:59:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-19T20:59:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:22.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:22.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:22.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:22.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-19T20:59:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-19T20:59:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:22.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:22.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:22.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:22.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:22.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:22.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:22.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:22.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:22.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:22.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:22.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:22.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:22.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35426], [object_number=33, tag_tid=1, entry_tid=35425], [object_number=32, tag_tid=1, entry_tid=35424], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:22.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:22.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-19T20:59:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-19T20:59:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:22.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:22.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35426], [object_number=33, tag_tid=1, entry_tid=35425], [object_number=32, tag_tid=1, entry_tid=35424], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35426' '!=' '' 2026-02-19T20:59:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:59:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35426' = '' 2026-02-19T20:59:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:59:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:22.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:59:22.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:22.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:59:22.593 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:59:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:59:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:59:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:59:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=35426 2026-02-19T20:59:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 35426 2026-02-19T20:59:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_33 2026-02-19T20:59:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_33 ']' 2026-02-19T20:59:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-19T20:59:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-19T20:59:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-19T20:59:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:22.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:22.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:22.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:22.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-19T20:59:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:22.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:22.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:22.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:22.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:22.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:22.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:22.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:22.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-19T20:59:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:22.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:22.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:22.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:22.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:23.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:23.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-19T20:59:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-19T20:59:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:23.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:23.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:23.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:23.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35426], [object_number=33, tag_tid=1, entry_tid=35425], [object_number=32, tag_tid=1, entry_tid=35424], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:23.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-19T20:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-19T20:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35426], [object_number=33, tag_tid=1, entry_tid=35425], [object_number=32, tag_tid=1, entry_tid=35424], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35426' '!=' 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:59:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_33 2026-02-19T20:59:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_33 ']' 2026-02-19T20:59:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-19T20:59:23.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-19T20:59:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:23.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:23.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:23.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-19T20:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:23.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:23.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:23.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:23.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:23.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:23.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:23.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-19T20:59:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:23.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:23.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:23.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:23.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:23.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:23.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:23.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35426], [object_number=33, tag_tid=1, entry_tid=35425], [object_number=32, tag_tid=1, entry_tid=35424], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-19T20:59:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-19T20:59:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:23.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:23.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:23.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35426], [object_number=33, tag_tid=1, entry_tid=35425], [object_number=32, tag_tid=1, entry_tid=35424], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35426' '!=' 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:59:24.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_33 2026-02-19T20:59:24.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_33 ']' 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:24.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:24.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:24.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:24.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:24.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-19T20:59:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:24.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:24.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:24.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:24.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:24.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-19T20:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:24.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:24.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:24.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:24.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:24.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-19T20:59:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-19T20:59:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35426], [object_number=33, tag_tid=1, entry_tid=35425], [object_number=32, tag_tid=1, entry_tid=35424], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35426], [object_number=33, tag_tid=1, entry_tid=35425], [object_number=32, tag_tid=1, entry_tid=35424], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35426' '!=' 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:59:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_33 2026-02-19T20:59:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_33 ']' 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:26.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:26.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:26.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:26.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-19T20:59:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:26.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:26.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:26.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-19T20:59:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-19T20:59:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:26.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:26.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35426], [object_number=33, tag_tid=1, entry_tid=35425], [object_number=32, tag_tid=1, entry_tid=35424], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:26.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:26.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:26.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-19T20:59:26.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:26.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:26.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:26.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-19T20:59:26.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:26.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:26.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:26.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:26.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:26.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:26.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:26.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35426], [object_number=33, tag_tid=1, entry_tid=35425], [object_number=32, tag_tid=1, entry_tid=35424], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:26.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:26.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:26.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35426' '!=' 'object_number=34, tag_tid=1, entry_tid=35426' 2026-02-19T20:59:26.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:26.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:59:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_33 2026-02-19T20:59:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_33 ']' 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:28.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:28.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:28.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:28.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:28.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:28.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:28.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:28.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-19T20:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-19T20:59:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-19T20:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:28.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:28.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:28.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:28.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:28.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-19T20:59:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:29.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-19T20:59:29.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-19T20:59:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:29.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:29.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:29.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-19T20:59:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-19T20:59:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T20:59:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-19T20:59:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-19T20:59:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-19T20:59:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35426], [object_number=35, tag_tid=1, entry_tid=35423]]], state=connected] 2026-02-19T20:59:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.status 2026-02-19T20:59:29.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:29.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:29.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:29.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2026-02-19T20:59:29.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:59:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2026-02-19T20:59:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_33': 2026-02-19T20:59:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 120cf4d3fd37 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.120cf4d3fd37 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:48:32 2026 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:48:32 2026 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:49:33 2026 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 120cf4d3fd37 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:59:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: bef284cd-9671-4ef0-8462-7cab326acb70 2026-02-19T20:59:30.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:59:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_33 snap 2026-02-19T20:59:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T20:59:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:59:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.export 2026-02-19T20:59:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_33.export 2026-02-19T20:59:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_33.export 2026-02-19T20:59:30.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.export 2026-02-19T20:59:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_33.export 2026-02-19T20:59:30.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_33.export 2026-02-19T20:59:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_33.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_33.export 2026-02-19T20:59:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:59:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_34 2026-02-19T20:59:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_34 snap 2026-02-19T20:59:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2026-02-19T20:59:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:59:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:59:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:59:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2026-02-19T20:59:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:59:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:59:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:59:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:59:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:59:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:59:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2026-02-19T20:59:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:59:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:59:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:59:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:33.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2026-02-19T20:59:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:59:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_34 2026-02-19T20:59:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:59:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:59:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:59:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:59:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_34 2026-02-19T20:59:34.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:34.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:59:34.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_34 2026-02-19T20:59:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:59:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:59:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:34.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_34 2026-02-19T20:59:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:59:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:59:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:59:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:59:34.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:59:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:59:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:59:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_34 2026-02-19T20:59:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:34.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:59:34.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_34 2026-02-19T20:59:34.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:34.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:34.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:34.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:59:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:59:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_34 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_34 ']' 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-19T20:59:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:34.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:34.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:34.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:34.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-19T20:59:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:34.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:34.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-19T20:59:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:34.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:34.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:34.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:34.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:34.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:34.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:34.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:34.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-19T20:59:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-19T20:59:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:34.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:34.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:34.572 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:34.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:34.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-19T20:59:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-19T20:59:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:34.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:34.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:34.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:34.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:34.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:34.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:34.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:34.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:34.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:34.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=1, entry_tid=40464' '!=' '' 2026-02-19T20:59:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:59:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=1, entry_tid=40464' = '' 2026-02-19T20:59:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:59:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:59:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:59:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:59:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:34.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:59:34.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:34.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:59:34.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:34.625 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:59:34.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40464 2026-02-19T20:59:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 40464 2026-02-19T20:59:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_34 2026-02-19T20:59:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_34 ']' 2026-02-19T20:59:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-19T20:59:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-19T20:59:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:34.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:34.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:34.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:34.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-19T20:59:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:34.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:34.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:34.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:34.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:34.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:34.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:34.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:34.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:34.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-19T20:59:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:34.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-19T20:59:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-19T20:59:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:35.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:35.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:35.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:35.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:35.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=1, entry_tid=40464' '!=' 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T20:59:35.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_34 2026-02-19T20:59:35.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:35.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:35.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:35.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:35.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_34 ']' 2026-02-19T20:59:35.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-19T20:59:35.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-19T20:59:35.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:35.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:35.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:35.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:35.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:35.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:35.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-19T20:59:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:35.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:35.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:35.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:35.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:35.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-19T20:59:35.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:35.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:35.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:35.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-19T20:59:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:35.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:35.685 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:35.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:35.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:35.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:35.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:35.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-19T20:59:35.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:35.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:35.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:35.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-19T20:59:35.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:35.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:35.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:35.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:35.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:35.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:35.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:35.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:35.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:35.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:35.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-19T20:59:35.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:35.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:35.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:35.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-19T20:59:35.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:35.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:35.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:35.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:35.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:35.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:35.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:35.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:35.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=1, entry_tid=40464' '!=' 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_34 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_34 ']' 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:36.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:36.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:36.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:36.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:36.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:36.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:36.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:36.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:36.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-19T20:59:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:36.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:36.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:36.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:36.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:36.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:36.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:36.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-19T20:59:36.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:36.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:36.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-19T20:59:36.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:36.861 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-19T20:59:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:36.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:36.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-19T20:59:36.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:36.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:36.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:36.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-19T20:59:36.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:36.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:36.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:36.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:36.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:36.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:36.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:36.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:36.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:36.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:37.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=1, entry_tid=40464' '!=' 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T20:59:38.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_34 2026-02-19T20:59:38.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:38.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_34 ']' 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:38.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:38.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:38.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:38.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:38.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-19T20:59:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:38.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:38.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-19T20:59:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-19T20:59:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-19T20:59:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:38.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:38.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:38.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:38.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-19T20:59:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-19T20:59:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:39.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:39.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:39.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:39.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:39.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=1, entry_tid=40464' '!=' 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:59:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_34 2026-02-19T20:59:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_34 ']' 2026-02-19T20:59:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-19T20:59:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-19T20:59:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-19T20:59:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:41.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:41.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:41.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:41.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:41.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:41.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:41.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:41.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:41.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:41.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-19T20:59:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:41.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:41.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:41.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:41.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:41.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-19T20:59:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-19T20:59:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:41.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:41.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:41.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:41.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:41.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-19T20:59:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-19T20:59:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=36, tag_tid=1, entry_tid=40464], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462], [object_number=37, tag_tid=1, entry_tid=40461]]], state=connected] 2026-02-19T20:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=36, tag_tid=1, entry_tid=40464' '!=' 'object_number=36, tag_tid=1, entry_tid=40464' 2026-02-19T20:59:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T20:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_34 2026-02-19T20:59:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_34 ']' 2026-02-19T20:59:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-19T20:59:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-19T20:59:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-19T20:59:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:43.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:43.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:43.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2026-02-19T20:59:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:43.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:43.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:43.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:43.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:43.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-19T20:59:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_34 2026-02-19T20:59:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_34 2026-02-19T20:59:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-19T20:59:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:43.594 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-19T20:59:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-19T20:59:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-19T20:59:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T20:59:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-19T20:59:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-19T20:59:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-19T20:59:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:43.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T20:59:43.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T20:59:43.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:43.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:43.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=40463], [object_number=38, tag_tid=1, entry_tid=40462]]], state=connected] 2026-02-19T20:59:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.status 2026-02-19T20:59:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T20:59:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2026-02-19T20:59:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:43.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:59:43.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2026-02-19T20:59:44.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_34': 2026-02-19T20:59:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:59:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:59:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:59:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 120cec1e579 2026-02-19T20:59:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.120cec1e579 2026-02-19T20:59:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:59:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:59:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:59:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:59:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:48:42 2026 2026-02-19T20:59:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:48:42 2026 2026-02-19T20:59:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:49:43 2026 2026-02-19T20:59:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:59:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 120cec1e579 2026-02-19T20:59:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:59:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:59:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 6f2c55b1-a7ad-4c69-b963-7c4df576e4a7 2026-02-19T20:59:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:59:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_34 snap 2026-02-19T20:59:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T20:59:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:59:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.export 2026-02-19T20:59:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_34.export 2026-02-19T20:59:44.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_34.export 2026-02-19T20:59:44.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.export 2026-02-19T20:59:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:44.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_34.export 2026-02-19T20:59:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_34.export 2026-02-19T20:59:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_34.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_34.export 2026-02-19T20:59:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:59:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_35 2026-02-19T20:59:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_35 snap 2026-02-19T20:59:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:59:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:59:49.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2026-02-19T20:59:49.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:49.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:49.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:59:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2026-02-19T20:59:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:59:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:59:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:59:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:50.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:59:50.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:59:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:59:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:50.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:50.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:50.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2026-02-19T20:59:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:59:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:59:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:59:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:59:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:59:50.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_35 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_35 2026-02-19T20:59:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:50.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:50.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:50.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:50.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:59:50.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:59:50.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:59:50.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:50.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:50.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_35 2026-02-19T20:59:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:59:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_35 ']' 2026-02-19T20:59:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-19T20:59:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-19T20:59:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:51.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:51.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:51.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:51.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:51.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:51.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:51.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:51.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-19T20:59:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-19T20:59:51.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:51.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-19T20:59:51.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:51.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:51.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_35 2026-02-19T20:59:51.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-19T20:59:51.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:51.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:51.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_35 2026-02-19T20:59:51.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:51.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-19T20:59:51.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-19T20:59:51.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-19T20:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:51.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:51.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:51.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37119], [object_number=34, tag_tid=1, entry_tid=37118]]], state=connected] 2026-02-19T20:59:51.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=1, entry_tid=37119], [object_number=34, tag_tid=1, entry_tid=37118], [object_number=33, tag_tid=1, entry_tid=37117], [object_number=32, tag_tid=1, entry_tid=37116]]], state=connected] 2026-02-19T20:59:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:51.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-19T20:59:51.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:51.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_35 2026-02-19T20:59:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-19T20:59:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_35 2026-02-19T20:59:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-19T20:59:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-19T20:59:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-19T20:59:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37119], [object_number=34, tag_tid=1, entry_tid=37118]]], state=connected] 2026-02-19T20:59:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=35, tag_tid=1, entry_tid=37119], [object_number=34, tag_tid=1, entry_tid=37118], [object_number=33, tag_tid=1, entry_tid=37117], [object_number=32, tag_tid=1, entry_tid=37116]]], state=connected] 2026-02-19T20:59:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=35, tag_tid=1, entry_tid=37119' 2026-02-19T20:59:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=35, tag_tid=1, entry_tid=37119' 2026-02-19T20:59:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=1, entry_tid=37119' '!=' '' 2026-02-19T20:59:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T20:59:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=35, tag_tid=1, entry_tid=37119' = '' 2026-02-19T20:59:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=35, tag_tid=1, entry_tid=37119' 2026-02-19T20:59:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:59:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T20:59:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=1, entry_tid=37119' 2026-02-19T20:59:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T20:59:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T20:59:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:59:51.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:51.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T20:59:51.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=35, tag_tid=1, entry_tid=37119' 2026-02-19T20:59:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T20:59:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T20:59:51.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=37119 2026-02-19T20:59:51.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 37119 2026-02-19T20:59:51.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:51.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:51.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_35 2026-02-19T20:59:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:59:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_35 ']' 2026-02-19T20:59:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-19T20:59:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-19T20:59:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:51.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:51.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:51.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-19T20:59:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-19T20:59:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_35 2026-02-19T20:59:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-19T20:59:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_35 2026-02-19T20:59:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-19T20:59:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-19T20:59:51.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:51.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-19T20:59:51.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:51.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:51.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:51.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37119], [object_number=34, tag_tid=1, entry_tid=37118]]], state=connected] 2026-02-19T20:59:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37119], [object_number=34, tag_tid=1, entry_tid=37118]]], state=connected] 2026-02-19T20:59:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-19T20:59:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_35 2026-02-19T20:59:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-19T20:59:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_35 2026-02-19T20:59:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-19T20:59:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-19T20:59:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:51.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-19T20:59:51.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T20:59:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T20:59:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37119], [object_number=34, tag_tid=1, entry_tid=37118]]], state=connected] 2026-02-19T20:59:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37119], [object_number=34, tag_tid=1, entry_tid=37118]]], state=connected] 2026-02-19T20:59:51.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.status 2026-02-19T20:59:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T20:59:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2026-02-19T20:59:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:59:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:59:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2026-02-19T20:59:52.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_35': 2026-02-19T20:59:52.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:59:52.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:59:52.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:59:52.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 120cdc2b7a7f 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.120cdc2b7a7f 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:48:54 2026 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:48:54 2026 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:49:55 2026 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 120cdc2b7a7f 2026-02-19T20:59:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:59:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:59:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 213f4ca2-ff6a-45d6-aa20-209a9122ef88 2026-02-19T20:59:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:59:52.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:52.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_35 snap 2026-02-19T20:59:52.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:52.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T20:59:52.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:52.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:59:52.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.export 2026-02-19T20:59:52.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_35.export 2026-02-19T20:59:52.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_35.export 2026-02-19T20:59:52.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.export 2026-02-19T20:59:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:52.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_35.export 2026-02-19T20:59:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_35.export 2026-02-19T20:59:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_35.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_35.export 2026-02-19T20:59:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:59:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_36 2026-02-19T20:59:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_36 snap 2026-02-19T20:59:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:59:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T20:59:55.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2026-02-19T20:59:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:59:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2026-02-19T20:59:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:59:55.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T20:59:55.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:55.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:55.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2026-02-19T20:59:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:59:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T20:59:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T20:59:56.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T20:59:56.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2026-02-19T20:59:56.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T20:59:56.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T20:59:56.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:56.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:56.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T20:59:56.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T20:59:56.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:56.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T20:59:56.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:56.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T20:59:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-19T20:59:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T20:59:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T20:59:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T20:59:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T20:59:56.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-19T20:59:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-19T20:59:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:59:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:56.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:56.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T20:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T20:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2026-02-19T20:59:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-19T20:59:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:59:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_36 2026-02-19T20:59:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:57.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:57.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T20:59:57.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T20:59:57.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:57.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:57.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_36 2026-02-19T20:59:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-19T20:59:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T20:59:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_36 2026-02-19T20:59:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T20:59:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T20:59:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:59:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:59:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T20:59:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:59:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T20:59:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T20:59:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T20:59:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T20:59:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:59:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T20:59:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T20:59:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T20:59:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T20:59:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T20:59:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:59:57.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:59:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:59:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:59:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_36 2026-02-19T20:59:57.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T20:59:57.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T20:59:57.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T20:59:57.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T20:59:57.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T20:59:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T20:59:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T20:59:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_36 2026-02-19T20:59:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:59:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T20:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_36 2026-02-19T20:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T20:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T20:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T20:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T20:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T20:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T20:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T20:59:57.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T20:59:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_36 2026-02-19T20:59:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:59:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T20:59:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_36 ']' 2026-02-19T20:59:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_36' 2026-02-19T20:59:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2026-02-19T20:59:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T20:59:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T20:59:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T20:59:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T20:59:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T20:59:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T20:59:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T20:59:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2026-02-19T20:59:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T20:59:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T20:59:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T20:59:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:57.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T20:59:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T20:59:57.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T20:59:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T20:59:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2026-02-19T20:59:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-19T20:59:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:57.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_36 2026-02-19T20:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T20:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T20:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T20:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T20:59:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T20:59:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T20:59:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T20:59:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T20:59:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T20:59:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:57.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:57.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T20:59:57.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_36 2026-02-19T20:59:57.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-19T20:59:57.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:57.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:57.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_36 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T20:59:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T20:59:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T20:59:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T20:59:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T20:59:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T20:59:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T20:59:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T20:59:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_36 2026-02-19T20:59:57.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-19T20:59:57.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T20:59:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T20:59:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T20:59:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_36 2026-02-19T20:59:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T20:59:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T20:59:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T20:59:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T20:59:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T20:59:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T20:59:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T20:59:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T20:59:57.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:59:57.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:59:57.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T20:59:57.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_36 2026-02-19T20:59:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_36 2026-02-19T20:59:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_36 2026-02-19T20:59:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_36 '' 2026-02-19T20:59:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_36 2026-02-19T20:59:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T20:59:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-19T20:59:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-19T20:59:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.status 2026-02-19T20:59:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-19T20:59:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.status 2026-02-19T20:59:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:59:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:59:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45691], [object_number=42, tag_tid=1, entry_tid=45690]]], state=connected] 2026-02-19T20:59:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45691], [object_number=42, tag_tid=1, entry_tid=45690]]], state=connected] 2026-02-19T20:59:57.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.status 2026-02-19T20:59:57.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:59:57.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2026-02-19T20:59:57.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:57.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:57.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_36 2026-02-19T20:59:57.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2026-02-19T20:59:57.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T20:59:57.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T20:59:57.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_36 2026-02-19T20:59:57.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T20:59:57.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-19T20:59:57.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-19T20:59:57.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T20:59:57.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.status 2026-02-19T20:59:57.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-19T20:59:57.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.status 2026-02-19T20:59:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T20:59:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T20:59:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T20:59:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45691], [object_number=42, tag_tid=1, entry_tid=45690]]], state=connected] 2026-02-19T20:59:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45691], [object_number=42, tag_tid=1, entry_tid=45690]]], state=connected] 2026-02-19T20:59:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.status 2026-02-19T20:59:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:59:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T20:59:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2026-02-19T20:59:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T20:59:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:59:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:57.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T20:59:57.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T20:59:57.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T20:59:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2026-02-19T20:59:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_36': 2026-02-19T20:59:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T20:59:58.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 120c3fab38f7 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.120c3fab38f7 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:49:03 2026 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:49:03 2026 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:50:04 2026 2026-02-19T20:59:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T20:59:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 120c3fab38f7 2026-02-19T20:59:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T20:59:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T20:59:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 85d99ed8-1ae5-4a5e-94ca-27418c991bb9 2026-02-19T20:59:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T20:59:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_36 snap 2026-02-19T20:59:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T20:59:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T20:59:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.export 2026-02-19T20:59:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_36.export 2026-02-19T20:59:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_36.export 2026-02-19T20:59:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.export 2026-02-19T20:59:58.915 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5897380485292382) > weight (0.25) 2026-02-19T20:59:58.916 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5722007085694729) > weight (0.25) 2026-02-19T20:59:58.916 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.693130865048013) > weight (0.25) 2026-02-19T20:59:58.916 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6665183179598606) > weight (0.25) 2026-02-19T20:59:58.916 INFO:tasks.rbd_mirror_thrash:waiting for 88.7700668945375 secs before thrashing 2026-02-19T20:59:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_36.export 2026-02-19T20:59:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T20:59:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_36.export 2026-02-19T20:59:59.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_36.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_36.export 2026-02-19T20:59:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T20:59:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T20:59:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_37 2026-02-19T20:59:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_37 snap 2026-02-19T20:59:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T20:59:59.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T20:59:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T20:59:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T20:59:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:00:01.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2026-02-19T21:00:01.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:01.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2026-02-19T21:00:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:00:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:00:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:00:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2026-02-19T21:00:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:00:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:00:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:00:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2026-02-19T21:00:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:00:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:00:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:00:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:00:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:00:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:00:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2026-02-19T21:00:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:00:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:02.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2026-02-19T21:00:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-19T21:00:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:00:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_37 2026-02-19T21:00:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:00:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:00:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:00:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:03.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:03.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_37 2026-02-19T21:00:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:00:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:00:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:00:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:00:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_37 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:00:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_37 2026-02-19T21:00:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:00:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:00:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:00:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:00:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:00:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:00:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:00:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_37 2026-02-19T21:00:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_37 ']' 2026-02-19T21:00:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-19T21:00:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-19T21:00:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:03.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:03.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:03.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:03.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:03.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:03.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:03.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-19T21:00:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-19T21:00:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:03.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:03.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:03.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-19T21:00:03.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:03.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:03.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:03.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-19T21:00:03.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:03.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:03.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:03.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47926' '!=' '' 2026-02-19T21:00:03.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:00:03.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47926' = '' 2026-02-19T21:00:03.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:03.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:03.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:00:03.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:03.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:00:03.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:00:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:03.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:00:03.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:03.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:00:03.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:03.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:00:03.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:03.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:00:03.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=47926 2026-02-19T21:00:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 47926 2026-02-19T21:00:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:00:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:00:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_37 2026-02-19T21:00:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_37 ']' 2026-02-19T21:00:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-19T21:00:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-19T21:00:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-19T21:00:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:03.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:03.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:03.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:03.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:03.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-19T21:00:03.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-19T21:00:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:03.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:03.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:03.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-19T21:00:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-19T21:00:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:03.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:03.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:03.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:03.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-19T21:00:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-19T21:00:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:03.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:03.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:03.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:03.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:03.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:03.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:03.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47926' '!=' 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:00:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_37 2026-02-19T21:00:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_37 ']' 2026-02-19T21:00:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-19T21:00:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-19T21:00:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-19T21:00:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-19T21:00:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-19T21:00:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:04.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:04.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:04.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:04.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-19T21:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-19T21:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:04.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:04.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:04.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:04.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:04.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:04.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-19T21:00:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-19T21:00:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:04.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:04.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:04.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:04.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:04.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47926' '!=' 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:00:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_37 2026-02-19T21:00:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_37 ']' 2026-02-19T21:00:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-19T21:00:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-19T21:00:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-19T21:00:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:05.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:05.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:05.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-19T21:00:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-19T21:00:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:05.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:05.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:05.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:05.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:05.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-19T21:00:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-19T21:00:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:05.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:05.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:05.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47926' '!=' 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:05.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:05.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:00:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_37 2026-02-19T21:00:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_37 ']' 2026-02-19T21:00:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-19T21:00:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-19T21:00:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:07.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:07.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:07.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:07.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:07.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:07.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:07.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:07.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:07.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:07.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:07.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:07.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-19T21:00:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-19T21:00:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:07.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:07.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:07.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:07.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:07.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:07.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:07.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-19T21:00:07.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:07.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:07.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-19T21:00:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:07.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:07.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:07.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:07.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:07.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:07.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:07.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:07.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-19T21:00:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-19T21:00:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:07.413 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:07.414 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:07.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:07.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:07.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:07.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:07.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:07.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:07.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:07.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:07.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47926' '!=' 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:00:09.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_37 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_37 ']' 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:09.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:09.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-19T21:00:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-19T21:00:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:09.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-19T21:00:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:09.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:09.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:09.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-19T21:00:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-19T21:00:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:09.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:09.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:09.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:09.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:09.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:09.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-19T21:00:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-19T21:00:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:09.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:09.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:09.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:09.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:09.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47926' '!=' 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:09.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:00:11.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_37 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_37 ']' 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:11.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:11.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-19T21:00:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-19T21:00:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:11.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:11.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-19T21:00:11.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:11.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:11.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:11.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-19T21:00:11.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:11.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:11.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:11.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:11.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:11.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:11.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:11.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:11.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:11.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:11.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:11.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:11.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:11.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-19T21:00:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-19T21:00:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:11.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:11.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:11.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47926], [object_number=45, tag_tid=1, entry_tid=47925], [object_number=44, tag_tid=1, entry_tid=47924], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47926' '!=' 'object_number=46, tag_tid=1, entry_tid=47926' 2026-02-19T21:00:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 4 2026-02-19T21:00:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_37 2026-02-19T21:00:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_37 ']' 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:15.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:15.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:15.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:15.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-19T21:00:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:15.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:15.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:15.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:15.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:15.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:15.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:15.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:16.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-19T21:00:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:16.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-19T21:00:16.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-19T21:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:16.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-19T21:00:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-19T21:00:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:00:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-19T21:00:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-19T21:00:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-19T21:00:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47926], [object_number=47, tag_tid=1, entry_tid=47923]]], state=connected] 2026-02-19T21:00:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.status 2026-02-19T21:00:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2026-02-19T21:00:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:00:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:00:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:00:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2026-02-19T21:00:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_37': 2026-02-19T21:00:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:00:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 120c4c5c686d 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.120c4c5c686d 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:49:15 2026 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:49:15 2026 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:50:16 2026 2026-02-19T21:00:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:00:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 120c4c5c686d 2026-02-19T21:00:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:00:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:00:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 476b8ccd-9dbe-4762-b2da-c13d92b3bc09 2026-02-19T21:00:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:00:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_37 snap 2026-02-19T21:00:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:00:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:00:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.export 2026-02-19T21:00:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_37.export 2026-02-19T21:00:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_37.export 2026-02-19T21:00:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.export 2026-02-19T21:00:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:00:17.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_37.export 2026-02-19T21:00:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:00:17.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_37.export 2026-02-19T21:00:17.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_37.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_37.export 2026-02-19T21:00:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:00:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_38 2026-02-19T21:00:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_38 snap 2026-02-19T21:00:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:00:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:00:19.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2026-02-19T21:00:19.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:19.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:19.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:00:19.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2026-02-19T21:00:19.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:19.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:19.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:00:19.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:00:19.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:00:19.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:19.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:00:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2026-02-19T21:00:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:00:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:00:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:00:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:00:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2026-02-19T21:00:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:00:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:00:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:00:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:00:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:00:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:00:20.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:00:20.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:20.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:20.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:20.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:20.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2026-02-19T21:00:21.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2026-02-19T21:00:21.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:00:21.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_38 2026-02-19T21:00:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:00:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:00:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_38 2026-02-19T21:00:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2026-02-19T21:00:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:00:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_38 2026-02-19T21:00:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T21:00:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T21:00:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T21:00:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:21.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:21.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:21.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:21.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_38 2026-02-19T21:00:21.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:00:21.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:00:21.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:00:21.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:00:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:00:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:00:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:00:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_38 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_38 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:00:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:00:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:00:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:00:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:00:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_38 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_38 ']' 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:21.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:21.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:21.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:21.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:21.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:21.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:21.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:21.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:21.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:21.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:21.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-19T21:00:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2026-02-19T21:00:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:21.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:21.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_38 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:21.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:21.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:21.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_38 2026-02-19T21:00:21.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2026-02-19T21:00:21.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:21.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:21.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_38 2026-02-19T21:00:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:21.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:21.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_38 2026-02-19T21:00:21.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_38 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_38 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-19T21:00:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-19T21:00:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.status 2026-02-19T21:00:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-19T21:00:21.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.status 2026-02-19T21:00:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18599], [object_number=18, tag_tid=1, entry_tid=18598]]], state=connected] 2026-02-19T21:00:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18599], [object_number=18, tag_tid=1, entry_tid=18598]]], state=connected] 2026-02-19T21:00:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.status 2026-02-19T21:00:21.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-19T21:00:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_38 2026-02-19T21:00:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-19T21:00:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_38 2026-02-19T21:00:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-19T21:00:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-19T21:00:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.status 2026-02-19T21:00:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-19T21:00:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.status 2026-02-19T21:00:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18599], [object_number=18, tag_tid=1, entry_tid=18598]]], state=connected] 2026-02-19T21:00:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=18599], [object_number=18, tag_tid=1, entry_tid=18598]]], state=connected] 2026-02-19T21:00:21.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.status 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:00:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:00:21.689 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.48168407151450476) > weight (0.25) 2026-02-19T21:00:21.689 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-19T21:00:21.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-19T21:00:21.689 INFO:tasks.rbd_mirror_thrash:waiting for 5.059145516427596 secs before reviving daemons 2026-02-19T21:00:21.690 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:00:21.677+0000 7fdda149c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 98856) UID: 1000 2026-02-19T21:00:21.690 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:00:21.681+0000 7fdd9a48e640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x564d9b130000 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-19T21:00:21.690 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:00:21.681+0000 7fdd9a48e640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x564d9b130000 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-19T21:00:22.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2026-02-19T21:00:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_38': 2026-02-19T21:00:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:00:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:00:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:00:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 120cc2641a3 2026-02-19T21:00:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.120cc2641a3 2026-02-19T21:00:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:00:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:00:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:00:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:00:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:49:27 2026 2026-02-19T21:00:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:49:27 2026 2026-02-19T21:00:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:49:27 2026 2026-02-19T21:00:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:00:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 120cc2641a3 2026-02-19T21:00:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:00:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:00:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 620b0194-3915-4fb0-87d9-110d9f7b14db 2026-02-19T21:00:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:00:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_38 snap 2026-02-19T21:00:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:00:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:00:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.export 2026-02-19T21:00:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_38.export 2026-02-19T21:00:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_38.export 2026-02-19T21:00:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.export 2026-02-19T21:00:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:00:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_38.export 2026-02-19T21:00:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:00:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_38.export 2026-02-19T21:00:23.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_38.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_38.export 2026-02-19T21:00:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:00:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_39 2026-02-19T21:00:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_39 snap 2026-02-19T21:00:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:00:25.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2026-02-19T21:00:25.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:25.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:25.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:25.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2026-02-19T21:00:25.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:25.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:25.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:25.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:00:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:00:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:00:26.749 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-19T21:00:26.749 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T21:00:26.749 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-19T21:00:26.749 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-19T21:00:26.750 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-19T21:00:26.752 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-19T21:00:26.752 INFO:tasks.rbd_mirror_thrash:waiting for 81.44746628775277 secs before thrashing 2026-02-19T21:00:26.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2026-02-19T21:00:26.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:26.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:26.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:26.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:00:26.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:00:26.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:00:26.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2026-02-19T21:00:26.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:00:26.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:00:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:00:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:00:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:00:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:00:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:00:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:00:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2026-02-19T21:00:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:00:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:00:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:00:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:26.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:26.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:26.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2026-02-19T21:00:27.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:27.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:00:27.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_39 2026-02-19T21:00:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T21:00:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T21:00:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T21:00:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_39 2026-02-19T21:00:27.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:27.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:00:27.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:27.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:27.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_39 2026-02-19T21:00:27.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T21:00:27.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:27.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:27.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:00:27.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:00:27.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:27.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:27.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T21:00:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:00:27.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:27.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:27.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:27.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_39 2026-02-19T21:00:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:00:27.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:00:27.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:00:27.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:00:27.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:00:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:00:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:00:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_39 2026-02-19T21:00:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:00:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:00:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:00:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_39 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:00:27.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_39 2026-02-19T21:00:27.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_39 ']' 2026-02-19T21:00:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-19T21:00:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-19T21:00:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:27.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:27.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:27.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:27.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:27.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:27.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:27.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:27.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:27.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:27.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:27.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-19T21:00:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:27.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:27.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:27.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:27.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:27.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2026-02-19T21:00:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_39 2026-02-19T21:00:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:00:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:00:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:00:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:00:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:00:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:00:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-19T21:00:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-19T21:00:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:27.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:27.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=46342], [object_number=45, tag_tid=1, entry_tid=46341], [object_number=44, tag_tid=1, entry_tid=46340], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:27.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:27.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-19T21:00:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-19T21:00:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:27.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:27.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=46342], [object_number=45, tag_tid=1, entry_tid=46341], [object_number=44, tag_tid=1, entry_tid=46340], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:27.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:27.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:27.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=46342' '!=' '' 2026-02-19T21:00:27.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:00:27.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=46342' = '' 2026-02-19T21:00:27.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:00:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:27.749 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:00:27.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:27.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:00:27.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:00:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:00:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:00:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:00:27.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=46342 2026-02-19T21:00:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 46342 2026-02-19T21:00:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:00:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:00:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_39 2026-02-19T21:00:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_39 ']' 2026-02-19T21:00:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-19T21:00:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-19T21:00:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:27.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:27.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:27.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:27.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:27.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:27.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:27.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:28.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:28.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:28.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:28.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:28.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:28.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:28.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:28.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:28.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:28.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:28.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:28.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:28.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:28.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:28.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:28.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:28.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:28.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_39 2026-02-19T21:00:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:00:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:00:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:00:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:00:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:00:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:28.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-19T21:00:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:28.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-19T21:00:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=46342], [object_number=45, tag_tid=1, entry_tid=46341], [object_number=44, tag_tid=1, entry_tid=46340], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:28.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:28.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-19T21:00:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-19T21:00:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=46342], [object_number=45, tag_tid=1, entry_tid=46341], [object_number=44, tag_tid=1, entry_tid=46340], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:28.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=46342' '!=' 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_39 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_39 ']' 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:28.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-19T21:00:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:28.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:28.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:28.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:28.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-19T21:00:28.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:28.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:28.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:28.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:28.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:28.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:28.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:28.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:28.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:28.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:28.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:28.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:28.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:28.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_39 2026-02-19T21:00:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:00:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:00:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:00:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:28.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:00:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:00:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-19T21:00:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-19T21:00:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:29.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:29.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=46342], [object_number=45, tag_tid=1, entry_tid=46341], [object_number=44, tag_tid=1, entry_tid=46340], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:29.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:29.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-19T21:00:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-19T21:00:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=46342], [object_number=45, tag_tid=1, entry_tid=46341], [object_number=44, tag_tid=1, entry_tid=46340], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=46342' '!=' 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:00:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_39 2026-02-19T21:00:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_39 ']' 2026-02-19T21:00:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-19T21:00:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-19T21:00:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-19T21:00:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2026-02-19T21:00:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:30.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:30.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:30.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:30.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:30.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:30.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_39 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:00:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:00:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:00:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-19T21:00:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-19T21:00:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=46342], [object_number=45, tag_tid=1, entry_tid=46341], [object_number=44, tag_tid=1, entry_tid=46340], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:30.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:30.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-19T21:00:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:30.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-19T21:00:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:30.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=46342], [object_number=45, tag_tid=1, entry_tid=46341], [object_number=44, tag_tid=1, entry_tid=46340], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=46342' '!=' 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:00:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_39 2026-02-19T21:00:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_39 ']' 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-19T21:00:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:31.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2026-02-19T21:00:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_39 2026-02-19T21:00:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:00:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:00:32.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:00:32.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:00:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-19T21:00:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-19T21:00:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:32.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:32.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:32.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=46342], [object_number=45, tag_tid=1, entry_tid=46341], [object_number=44, tag_tid=1, entry_tid=46340], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-19T21:00:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-19T21:00:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=46342], [object_number=45, tag_tid=1, entry_tid=46341], [object_number=44, tag_tid=1, entry_tid=46340], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=46342' '!=' 'object_number=46, tag_tid=1, entry_tid=46342' 2026-02-19T21:00:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:00:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_39 2026-02-19T21:00:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:34.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_39 ']' 2026-02-19T21:00:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-19T21:00:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-19T21:00:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-19T21:00:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:34.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:34.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:34.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-19T21:00:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:34.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2026-02-19T21:00:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:34.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:34.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-19T21:00:34.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:34.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_39 2026-02-19T21:00:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:00:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:00:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:00:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:00:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:00:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:00:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:00:34.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:34.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:34.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:00:34.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:00:34.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:00:34.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:34.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:34.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:00:34.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_39 2026-02-19T21:00:34.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-19T21:00:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-19T21:00:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:34.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:34.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:34.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:34.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:34.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-19T21:00:34.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:34.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:34.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:34.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-19T21:00:34.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:34.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:34.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:00:34.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:34.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-19T21:00:34.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-19T21:00:34.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:34.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:34.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-19T21:00:34.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:00:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:00:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=46342], [object_number=47, tag_tid=1, entry_tid=46339]]], state=connected] 2026-02-19T21:00:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.status 2026-02-19T21:00:34.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:34.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:00:34.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2026-02-19T21:00:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:00:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:00:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:00:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2026-02-19T21:00:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_39': 2026-02-19T21:00:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:00:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:00:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:00:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 120c644a73c8 2026-02-19T21:00:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.120c644a73c8 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:49:34 2026 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:49:34 2026 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:51:56 2026 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 120c644a73c8 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:00:35.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:00:35.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 3d6834fd-87fd-45ae-b3cc-e81d7fb671d2 2026-02-19T21:00:35.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:00:35.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:35.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_39 snap 2026-02-19T21:00:35.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:35.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:00:35.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:35.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:00:35.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.export 2026-02-19T21:00:35.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_39.export 2026-02-19T21:00:35.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_39.export 2026-02-19T21:00:35.646 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.export 2026-02-19T21:00:35.792 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:00:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_39.export 2026-02-19T21:00:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:00:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_39.export 2026-02-19T21:00:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_39.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_39.export 2026-02-19T21:00:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:00:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_40 2026-02-19T21:00:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_40 snap 2026-02-19T21:00:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:00:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2026-02-19T21:00:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2026-02-19T21:00:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:00:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:00:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:00:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2026-02-19T21:00:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:00:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:00:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:00:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2026-02-19T21:00:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:00:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:00:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:00:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:00:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:00:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:00:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2026-02-19T21:00:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:00:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:38.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:38.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:00:38.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:38.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:38.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:38.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:38.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:38.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:38.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:38.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:00:38.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:00:38.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:38.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:38.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:38.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:38.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2026-02-19T21:00:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:00:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:00:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:00:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:00:38.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:00:38.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:00:38.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_40 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_40 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:00:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:00:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:00:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:00:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:00:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:38.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:00:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_40 2026-02-19T21:00:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_40 ']' 2026-02-19T21:00:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-19T21:00:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-19T21:00:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-19T21:00:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:39.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:39.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:39.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:39.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:39.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-19T21:00:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-19T21:00:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-19T21:00:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:39.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:39.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:39.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:39.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-19T21:00:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-19T21:00:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:39.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:39.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:39.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40498' '!=' '' 2026-02-19T21:00:39.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:00:39.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40498' = '' 2026-02-19T21:00:39.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:00:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:39.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:00:39.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:39.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:00:39.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:39.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:00:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:00:39.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:00:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:00:39.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=40498 2026-02-19T21:00:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 40498 2026-02-19T21:00:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:00:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:00:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_40 2026-02-19T21:00:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_40 ']' 2026-02-19T21:00:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-19T21:00:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-19T21:00:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-19T21:00:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:39.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-19T21:00:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-19T21:00:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-19T21:00:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:39.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:39.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:39.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:39.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:39.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:39.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-19T21:00:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-19T21:00:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:39.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:39.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:39.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:39.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40498' '!=' 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:00:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_40 2026-02-19T21:00:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_40 ']' 2026-02-19T21:00:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-19T21:00:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-19T21:00:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-19T21:00:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:40.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-19T21:00:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:40.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:40.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:40.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:40.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:40.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-19T21:00:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-19T21:00:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40498' '!=' 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:40.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:40.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:00:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_40 2026-02-19T21:00:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_40 ']' 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-19T21:00:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-19T21:00:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:41.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-19T21:00:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-19T21:00:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:41.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40498' '!=' 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:00:42.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_40 2026-02-19T21:00:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_40 ']' 2026-02-19T21:00:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-19T21:00:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-19T21:00:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-19T21:00:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:42.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-19T21:00:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-19T21:00:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-19T21:00:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:42.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:42.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-19T21:00:42.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-19T21:00:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:42.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:42.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40498' '!=' 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:43.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:43.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:00:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_40 2026-02-19T21:00:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_40 ']' 2026-02-19T21:00:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-19T21:00:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-19T21:00:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-19T21:00:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:45.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-19T21:00:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-19T21:00:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-19T21:00:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:45.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:45.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:45.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-19T21:00:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-19T21:00:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=40498], [object_number=37, tag_tid=1, entry_tid=40497], [object_number=36, tag_tid=1, entry_tid=40496], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=40498' '!=' 'object_number=38, tag_tid=1, entry_tid=40498' 2026-02-19T21:00:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:00:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_40 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_40 ']' 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:47.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-19T21:00:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-19T21:00:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-19T21:00:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:47.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:47.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:47.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:47.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:47.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:47.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:47.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-19T21:00:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-19T21:00:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:00:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-19T21:00:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-19T21:00:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-19T21:00:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:00:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:00:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=40498], [object_number=39, tag_tid=1, entry_tid=40495]]], state=connected] 2026-02-19T21:00:47.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.status 2026-02-19T21:00:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:00:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2026-02-19T21:00:47.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:47.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:47.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:47.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:47.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:00:47.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:00:47.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:00:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2026-02-19T21:00:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_40': 2026-02-19T21:00:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:00:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:00:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:00:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 121c2445c088 2026-02-19T21:00:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.121c2445c088 2026-02-19T21:00:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:00:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:00:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:00:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:00:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:49:45 2026 2026-02-19T21:00:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:49:45 2026 2026-02-19T21:00:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:50:46 2026 2026-02-19T21:00:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:00:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 121c2445c088 2026-02-19T21:00:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:00:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:00:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 5933144f-ff27-49e9-8932-6fa58236c407 2026-02-19T21:00:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:00:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_40 snap 2026-02-19T21:00:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:00:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:00:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.export 2026-02-19T21:00:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_40.export 2026-02-19T21:00:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_40.export 2026-02-19T21:00:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.export 2026-02-19T21:00:48.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:00:48.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_40.export 2026-02-19T21:00:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:00:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_40.export 2026-02-19T21:00:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_40.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_40.export 2026-02-19T21:00:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:00:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_41 2026-02-19T21:00:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_41 snap 2026-02-19T21:00:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:00:50.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2026-02-19T21:00:50.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:50.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2026-02-19T21:00:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:00:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:00:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:00:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:00:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:00:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:00:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:51.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:00:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2026-02-19T21:00:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:00:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:00:51.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2026-02-19T21:00:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:00:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_41 2026-02-19T21:00:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:00:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:51.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:51.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:51.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:00:51.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:00:51.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:51.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:51.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_41 2026-02-19T21:00:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:00:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_41 2026-02-19T21:00:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T21:00:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:00:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T21:00:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T21:00:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T21:00:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_41 2026-02-19T21:00:52.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:00:52.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:00:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:00:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:00:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:00:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:00:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:00:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_41 2026-02-19T21:00:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:00:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:00:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:00:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:00:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_41 2026-02-19T21:00:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:00:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:00:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:00:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:00:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:00:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:00:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:00:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_41 2026-02-19T21:00:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_41 ']' 2026-02-19T21:00:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-19T21:00:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-19T21:00:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-19T21:00:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:52.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:52.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-19T21:00:52.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:52.422 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:52.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:52.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:52.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:52.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:52.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:52.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:52.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:52.492 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-19T21:00:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:52.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-19T21:00:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-19T21:00:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:52.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:52.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:52.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:52.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:52.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17748], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746], [object_number=17, tag_tid=1, entry_tid=17745]]], state=connected] 2026-02-19T21:00:52.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-19T21:00:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-19T21:00:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-19T21:00:52.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-19T21:00:52.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-19T21:00:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:52.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:52.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:52.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:52.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:52.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17748], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746], [object_number=17, tag_tid=1, entry_tid=17745]]], state=connected] 2026-02-19T21:00:52.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:52.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:52.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:52.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=17748' '!=' '' 2026-02-19T21:00:52.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:00:52.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=17748' = '' 2026-02-19T21:00:52.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:52.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:52.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:00:52.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:00:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:52.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:00:52.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:52.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:00:52.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:52.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:00:52.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:52.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:00:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:00:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:00:52.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=17748 2026-02-19T21:00:52.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 17748 2026-02-19T21:00:52.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:00:52.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:52.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:00:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_41 2026-02-19T21:00:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_41 ']' 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-19T21:00:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:52.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:52.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:52.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:52.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:52.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:52.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-19T21:00:52.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:52.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:52.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-19T21:00:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:52.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-19T21:00:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-19T21:00:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-19T21:00:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-19T21:00:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-19T21:00:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:53.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:53.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:53.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:53.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17748], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746], [object_number=17, tag_tid=1, entry_tid=17745]]], state=connected] 2026-02-19T21:00:53.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-19T21:00:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-19T21:00:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-19T21:00:53.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-19T21:00:53.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:53.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-19T21:00:53.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17748], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746], [object_number=17, tag_tid=1, entry_tid=17745]]], state=connected] 2026-02-19T21:00:53.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=17748' '!=' 'object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:00:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_41 2026-02-19T21:00:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_41 ']' 2026-02-19T21:00:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-19T21:00:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-19T21:00:53.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:53.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:53.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:53.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:53.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:53.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:53.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:53.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-19T21:00:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:53.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:53.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:53.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:53.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-19T21:00:53.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:53.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:53.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:53.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-19T21:00:53.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:53.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:53.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:53.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:53.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-19T21:00:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-19T21:00:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-19T21:00:53.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-19T21:00:53.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:53.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-19T21:00:53.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:53.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:53.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:53.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:53.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17748], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746], [object_number=17, tag_tid=1, entry_tid=17745]]], state=connected] 2026-02-19T21:00:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-19T21:00:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:53.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-19T21:00:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-19T21:00:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-19T21:00:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-19T21:00:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17748], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746], [object_number=17, tag_tid=1, entry_tid=17745]]], state=connected] 2026-02-19T21:00:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:53.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:53.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:53.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=17748' '!=' 'object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:53.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:53.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:00:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_41 2026-02-19T21:00:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_41 ']' 2026-02-19T21:00:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-19T21:00:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-19T21:00:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:54.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:54.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:54.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:54.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-19T21:00:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:54.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:54.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:54.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:54.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:54.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:54.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:54.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-19T21:00:54.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:54.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:54.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:54.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:54.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:54.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:54.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:54.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:54.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:54.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:54.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:54.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:54.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-19T21:00:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-19T21:00:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:54.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-19T21:00:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-19T21:00:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:54.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:54.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-19T21:00:54.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:54.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:54.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:54.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:54.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:54.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17748], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746], [object_number=17, tag_tid=1, entry_tid=17745]]], state=connected] 2026-02-19T21:00:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:54.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-19T21:00:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-19T21:00:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-19T21:00:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-19T21:00:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-19T21:00:54.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=17748], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746], [object_number=17, tag_tid=1, entry_tid=17745]]], state=connected] 2026-02-19T21:00:54.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=17748' '!=' 'object_number=16, tag_tid=1, entry_tid=17748' 2026-02-19T21:00:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:00:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:00:56.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_41 2026-02-19T21:00:56.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:56.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:56.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:56.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:00:56.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_41 ']' 2026-02-19T21:00:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-19T21:00:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-19T21:00:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:00:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:00:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:00:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:00:56.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:00:56.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:00:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:00:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-19T21:00:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:00:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:56.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:56.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:00:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:00:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:00:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:56.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:00:56.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:56.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-19T21:00:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:00:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:00:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:00:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:00:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:00:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:00:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:00:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:00:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:00:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:00:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:56.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:56.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:00:56.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-19T21:00:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:00:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:00:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:00:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-19T21:00:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:00:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:00:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:00:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:00:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:00:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:00:56.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:00:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:00:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:00:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-19T21:00:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-19T21:00:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-19T21:00:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:00:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-19T21:00:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-19T21:00:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-19T21:00:56.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:56.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:56.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-19T21:00:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-19T21:00:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:00:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:00:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:00:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:00:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-19T21:00:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-19T21:00:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:00:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:56.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-19T21:00:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 4 2026-02-19T21:00:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 4 2026-02-19T21:00:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:00:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=17747], [object_number=18, tag_tid=1, entry_tid=17746]]], state=connected] 2026-02-19T21:00:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.status 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:00:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:00:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2026-02-19T21:00:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_41': 2026-02-19T21:00:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 121cd4b4a758 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.121cd4b4a758 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:49:57 2026 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:49:57 2026 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:49:57 2026 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 121cd4b4a758 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: dee10364-a17e-4739-bab1-1e2c1583c379 2026-02-19T21:00:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:00:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_41 snap 2026-02-19T21:00:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:00:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:00:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.export 2026-02-19T21:00:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_41.export 2026-02-19T21:00:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_41.export 2026-02-19T21:00:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.export 2026-02-19T21:00:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:00:58.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_41.export 2026-02-19T21:00:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:00:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_41.export 2026-02-19T21:00:58.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_41.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_41.export 2026-02-19T21:00:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:00:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:00:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_42 2026-02-19T21:00:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_42 snap 2026-02-19T21:00:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:00:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:00:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:00:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:00:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:01:00.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2026-02-19T21:01:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2026-02-19T21:01:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:01:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:01:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:00.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:01:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2026-02-19T21:01:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:01:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:01:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:01:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2026-02-19T21:01:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:01:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:01:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:01:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:01:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:01:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:01:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:01:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:01:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2026-02-19T21:01:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:01:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_42 2026-02-19T21:01:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:02.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:01:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:01:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:02.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_42 2026-02-19T21:01:02.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:02.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:01:02.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_42 2026-02-19T21:01:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T21:01:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T21:01:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T21:01:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T21:01:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_42 2026-02-19T21:01:02.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:01:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:01:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:01:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:01:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:01:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:01:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:01:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_42 2026-02-19T21:01:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:01:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_42 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:01:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:01:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:01:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_42 2026-02-19T21:01:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_42 ']' 2026-02-19T21:01:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-19T21:01:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-19T21:01:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-19T21:01:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:02.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-02-19T21:01:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:02.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:02.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:02.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-02-19T21:01:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:02.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:02.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:02.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:02.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-19T21:01:02.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-19T21:01:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:02.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:02.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:02.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:02.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:02.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:02.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:02.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:02.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45537], [object_number=40, tag_tid=1, entry_tid=45536], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T21:01:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-19T21:01:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-19T21:01:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:02.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:02.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:02.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:02.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45537], [object_number=40, tag_tid=1, entry_tid=45536], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:02.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45537' '!=' '' 2026-02-19T21:01:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:01:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45537' = '' 2026-02-19T21:01:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T21:01:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:01:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:01:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:02.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:01:02.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:01:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T21:01:02.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:01:02.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:02.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:01:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:01:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=45537 2026-02-19T21:01:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 45537 2026-02-19T21:01:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:01:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:01:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_42 2026-02-19T21:01:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_42 ']' 2026-02-19T21:01:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-19T21:01:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-19T21:01:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:02.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:02.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:02.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:02.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:02.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-02-19T21:01:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-02-19T21:01:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:03.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:03.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:03.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-19T21:01:03.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:03.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:03.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:03.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-19T21:01:03.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:03.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:03.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:03.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:03.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:03.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:03.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:03.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:03.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45537], [object_number=40, tag_tid=1, entry_tid=45536], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T21:01:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-19T21:01:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-19T21:01:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:03.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45537], [object_number=40, tag_tid=1, entry_tid=45536], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45537' '!=' 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:01:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_42 2026-02-19T21:01:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_42 ']' 2026-02-19T21:01:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-19T21:01:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-19T21:01:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:03.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:03.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:03.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:03.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:03.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-02-19T21:01:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:03.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-19T21:01:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-19T21:01:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:03.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45537], [object_number=40, tag_tid=1, entry_tid=45536], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T21:01:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-19T21:01:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-19T21:01:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:03.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:03.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:03.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:03.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45537], [object_number=40, tag_tid=1, entry_tid=45536], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45537' '!=' 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:01:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_42 2026-02-19T21:01:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_42 ']' 2026-02-19T21:01:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-19T21:01:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-19T21:01:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:04.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-02-19T21:01:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:04.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:04.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:04.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:04.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:04.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:04.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:04.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:04.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:04.738 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-02-19T21:01:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:04.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:04.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-19T21:01:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-19T21:01:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:04.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45537], [object_number=40, tag_tid=1, entry_tid=45536], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:04.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45537], [object_number=40, tag_tid=1, entry_tid=45536], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45537' '!=' 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:01:06.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_42 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_42 ']' 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:06.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:06.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:06.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:06.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:06.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:06.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-02-19T21:01:06.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:06.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:06.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:06.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:06.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:06.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-19T21:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-19T21:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:06.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:06.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:06.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:06.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:06.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:06.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:06.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45537], [object_number=40, tag_tid=1, entry_tid=45536], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T21:01:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-19T21:01:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-19T21:01:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=45537], [object_number=40, tag_tid=1, entry_tid=45536], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=45537' '!=' 'object_number=41, tag_tid=1, entry_tid=45537' 2026-02-19T21:01:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:01:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_42 2026-02-19T21:01:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_42 ']' 2026-02-19T21:01:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-19T21:01:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-19T21:01:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-19T21:01:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:09.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:09.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:09.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2026-02-19T21:01:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-19T21:01:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_42 2026-02-19T21:01:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:09.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:09.211 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_42 2026-02-19T21:01:09.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:09.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-19T21:01:09.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:09.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-19T21:01:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:09.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:09.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:09.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:09.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:09.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:09.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T21:01:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-19T21:01:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-19T21:01:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:01:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-19T21:01:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-19T21:01:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-19T21:01:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:09.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 10 2026-02-19T21:01:09.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 10 2026-02-19T21:01:09.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:09.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:09.355 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=45535], [object_number=42, tag_tid=1, entry_tid=45534]]], state=connected] 2026-02-19T21:01:09.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.status 2026-02-19T21:01:09.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T21:01:09.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-19T21:01:09.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:09.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2026-02-19T21:01:09.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:09.359 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:09.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:09.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:09.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:01:09.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:01:09.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:01:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2026-02-19T21:01:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_42': 2026-02-19T21:01:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:01:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:01:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:01:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 121c669bdc09 2026-02-19T21:01:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.121c669bdc09 2026-02-19T21:01:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:01:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:01:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:01:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:01:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:50:03 2026 2026-02-19T21:01:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:50:03 2026 2026-02-19T21:01:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:51:04 2026 2026-02-19T21:01:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:01:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 121c669bdc09 2026-02-19T21:01:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:01:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:01:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: d10f0846-ad09-40df-9a15-77021c3921e0 2026-02-19T21:01:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:01:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_42 snap 2026-02-19T21:01:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:01:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:01:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.export 2026-02-19T21:01:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_42.export 2026-02-19T21:01:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_42.export 2026-02-19T21:01:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.export 2026-02-19T21:01:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:01:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_42.export 2026-02-19T21:01:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:01:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_42.export 2026-02-19T21:01:10.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_42.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_42.export 2026-02-19T21:01:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:01:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_43 2026-02-19T21:01:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_43 snap 2026-02-19T21:01:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:01:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2026-02-19T21:01:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2026-02-19T21:01:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:01:12.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:01:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:01:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2026-02-19T21:01:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:01:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:01:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:01:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2026-02-19T21:01:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:01:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:01:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:01:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:01:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:01:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:01:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-19T21:01:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:01:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:13.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:01:13.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:01:13.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:13.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:13.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:13.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:13.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-19T21:01:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:01:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:01:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:01:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_43 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_43 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:01:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:01:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:01:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:01:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:01:14.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_43 2026-02-19T21:01:14.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:14.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:14.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:14.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:14.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_43 ']' 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-19T21:01:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:14.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:14.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:14.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:14.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:14.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:14.219 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-19T21:01:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-19T21:01:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-19T21:01:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-19T21:01:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-19T21:01:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:14.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:14.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39306' '!=' '' 2026-02-19T21:01:14.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:01:14.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39306' = '' 2026-02-19T21:01:14.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:01:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:01:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:14.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:01:14.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:14.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:01:14.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:01:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:01:14.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:14.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:01:14.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:14.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=39306 2026-02-19T21:01:14.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 39306 2026-02-19T21:01:14.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:01:14.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:14.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:01:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_43 2026-02-19T21:01:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_43 ']' 2026-02-19T21:01:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-19T21:01:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-19T21:01:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:14.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:14.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:14.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:14.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:14.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-19T21:01:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-19T21:01:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-19T21:01:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:14.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:14.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:14.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:14.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:14.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:14.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:14.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:14.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39306' '!=' 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:14.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:14.747 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:01:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_43 2026-02-19T21:01:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_43 ']' 2026-02-19T21:01:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-19T21:01:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-19T21:01:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:15.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-19T21:01:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-19T21:01:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-19T21:01:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:15.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:15.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:15.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:15.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-19T21:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-19T21:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:15.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39306' '!=' 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:01:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_43 2026-02-19T21:01:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_43 ']' 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-19T21:01:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:16.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39306' '!=' 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_43 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_43 ']' 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-19T21:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:17.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:17.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:17.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:17.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:17.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:17.944 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:17.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:17.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:17.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:17.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:17.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:17.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:17.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:17.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-19T21:01:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-19T21:01:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:18.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:18.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:18.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:18.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:18.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:18.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:18.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39306' '!=' 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_43 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_43 ']' 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:20.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:20.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:20.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:20.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:20.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:20.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:20.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:20.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:20.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:20.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:20.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:20.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-19T21:01:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-19T21:01:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39306], [object_number=37, tag_tid=1, entry_tid=39305], [object_number=36, tag_tid=1, entry_tid=39304], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39306' '!=' 'object_number=38, tag_tid=1, entry_tid=39306' 2026-02-19T21:01:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:01:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_43 2026-02-19T21:01:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_43 ']' 2026-02-19T21:01:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-19T21:01:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-19T21:01:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:22.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:22.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:22.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:22.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:22.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:22.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:22.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-19T21:01:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:22.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:22.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:22.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:22.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-19T21:01:22.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:22.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-19T21:01:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:01:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-19T21:01:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-19T21:01:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:22.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:22.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-19T21:01:22.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 9 2026-02-19T21:01:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 9 2026-02-19T21:01:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39306], [object_number=39, tag_tid=1, entry_tid=39303]]], state=connected] 2026-02-19T21:01:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.status 2026-02-19T21:01:22.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:22.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-19T21:01:22.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:22.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2026-02-19T21:01:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:01:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:01:22.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:01:23.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2026-02-19T21:01:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_43': 2026-02-19T21:01:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:01:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:01:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:01:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 120c520ad079 2026-02-19T21:01:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.120c520ad079 2026-02-19T21:01:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:01:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:01:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:01:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:01:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:50:15 2026 2026-02-19T21:01:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:50:15 2026 2026-02-19T21:01:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:52:17 2026 2026-02-19T21:01:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:01:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 120c520ad079 2026-02-19T21:01:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:01:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:01:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 0aa6f22a-3c52-4b0e-921d-ba8b5c916f88 2026-02-19T21:01:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:01:23.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:23.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_43 snap 2026-02-19T21:01:23.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:23.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:01:23.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:01:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.export 2026-02-19T21:01:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_43.export 2026-02-19T21:01:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_43.export 2026-02-19T21:01:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.export 2026-02-19T21:01:23.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:01:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_43.export 2026-02-19T21:01:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:01:23.727 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_43.export 2026-02-19T21:01:23.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_43.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_43.export 2026-02-19T21:01:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:01:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_44 2026-02-19T21:01:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_44 snap 2026-02-19T21:01:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:23.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:01:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2026-02-19T21:01:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:26.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:01:26.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:26.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:01:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:26.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:26.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:26.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:26.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2026-02-19T21:01:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:01:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_44 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:01:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:27.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:27.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:27.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_44 2026-02-19T21:01:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:01:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_44 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T21:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T21:01:27.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:27.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:27.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:27.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:27.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_44 2026-02-19T21:01:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:01:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_44 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:3 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:01:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:3 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:3 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=3 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=3 2026-02-19T21:01:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:01:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_44 2026-02-19T21:01:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:01:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:01:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:01:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:01:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:01:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:01:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_44 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_44 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_44 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_44 ']' 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:27.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:27.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:27.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:27.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:27.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:27.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:01:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:01:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:27.687 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8730368770211538) > weight (0.25) 2026-02-19T21:01:27.687 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.863921564242267) > weight (0.25) 2026-02-19T21:01:27.687 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-19T21:01:27.687 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-19T21:01:27.688 INFO:tasks.rbd_mirror_thrash:waiting for 0.4245657945883752 secs before reviving daemons 2026-02-19T21:01:27.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:01:27.676+0000 7fa6720da640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 82198) UID: 1000 2026-02-19T21:01:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-19T21:01:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-19T21:01:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:27.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:27.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:27.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-19T21:01:27.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-19T21:01:27.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:27.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:27.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-19T21:01:27.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:01:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:01:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:27.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=24320], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318], [object_number=21, tag_tid=1, entry_tid=24317]]], state=connected] 2026-02-19T21:01:27.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:27.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:01:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-19T21:01:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:27.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-19T21:01:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-19T21:01:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-19T21:01:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:27.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:27.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-19T21:01:27.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:01:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:01:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=24320], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318], [object_number=21, tag_tid=1, entry_tid=24317]]], state=connected] 2026-02-19T21:01:27.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:27.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:27.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=24320' '!=' '' 2026-02-19T21:01:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:01:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=24320' = '' 2026-02-19T21:01:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:01:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:01:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:01:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:01:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:01:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:01:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:01:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:27.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:01:27.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:27.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:01:27.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=24320 2026-02-19T21:01:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 24320 2026-02-19T21:01:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:01:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:01:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_44 2026-02-19T21:01:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_44 ']' 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:28.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:28.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:28.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:28.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:28.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:28.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:28.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:28.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-19T21:01:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:28.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:28.113 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-19T21:01:28.113 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T21:01:28.113 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-19T21:01:28.113 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-19T21:01:28.113 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-19T21:01:28.114 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-19T21:01:28.114 INFO:tasks.rbd_mirror_thrash:waiting for 113.76117488734715 secs before thrashing 2026-02-19T21:01:28.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:28.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:28.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:28.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:28.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:28.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:28.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:28.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:28.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:28.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:28.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:28.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:28.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:28.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-19T21:01:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:01:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:01:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:01:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:01:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-19T21:01:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-19T21:01:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:28.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-19T21:01:28.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-19T21:01:28.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:28.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-19T21:01:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:01:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:01:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=24320], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318], [object_number=21, tag_tid=1, entry_tid=24317]]], state=connected] 2026-02-19T21:01:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:01:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-19T21:01:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-19T21:01:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-19T21:01:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-19T21:01:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-19T21:01:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:01:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:01:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=24320], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318], [object_number=21, tag_tid=1, entry_tid=24317]]], state=connected] 2026-02-19T21:01:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=24320' '!=' 'object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:01:28.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_44 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_44 ']' 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:28.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:28.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:28.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:28.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:28.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-19T21:01:28.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:28.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:28.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:28.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:28.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:28.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:28.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-19T21:01:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:01:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:28.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:01:28.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:01:28.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:28.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:28.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:28.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:01:28.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:01:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:01:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:01:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:01:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:29.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-19T21:01:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-19T21:01:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-19T21:01:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-19T21:01:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-19T21:01:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:01:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:01:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=24320], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318], [object_number=21, tag_tid=1, entry_tid=24317]]], state=connected] 2026-02-19T21:01:29.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:01:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-19T21:01:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-19T21:01:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-19T21:01:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-19T21:01:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-19T21:01:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:01:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:01:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=24320], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318], [object_number=21, tag_tid=1, entry_tid=24317]]], state=connected] 2026-02-19T21:01:29.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:29.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:29.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:29.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=24320' '!=' 'object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:29.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:29.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_44 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_44 ']' 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:30.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:30.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:30.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-19T21:01:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:30.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:30.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:30.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:30.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:30.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:30.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:30.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:30.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:30.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:30.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:30.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:30.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:30.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:30.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:30.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:30.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-19T21:01:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:01:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:01:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:01:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:01:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:01:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:01:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:01:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-19T21:01:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-19T21:01:30.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:30.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:30.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:30.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:30.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-19T21:01:30.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-19T21:01:30.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:30.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-19T21:01:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:01:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:01:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=24320], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318], [object_number=21, tag_tid=1, entry_tid=24317]]], state=connected] 2026-02-19T21:01:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:01:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-19T21:01:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-19T21:01:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:30.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-19T21:01:30.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-19T21:01:30.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:30.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:30.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-19T21:01:30.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:01:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:01:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:30.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:30.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=20, tag_tid=1, entry_tid=24320], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318], [object_number=21, tag_tid=1, entry_tid=24317]]], state=connected] 2026-02-19T21:01:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=20, tag_tid=1, entry_tid=24320' '!=' 'object_number=20, tag_tid=1, entry_tid=24320' 2026-02-19T21:01:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:01:31.243 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:01:31.239+0000 7fe4dd68c640 -1 reset not still connected to 0x564a120425b0 2026-02-19T21:01:31.243 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:01:31.239+0000 7fe4dd68c640 -1 reset not still connected to 0x564a12043a00 2026-02-19T21:01:31.244 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:01:31.239+0000 7fe4dd68c640 -1 reset not still connected to 0x564a35f351e0 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_44 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_44 ']' 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-19T21:01:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:32.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:32.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:32.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:32.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_44 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:32.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-19T21:01:32.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:32.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_44 2026-02-19T21:01:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:01:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:01:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:01:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:01:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_44 2026-02-19T21:01:32.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:32.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-19T21:01:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-19T21:01:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-19T21:01:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-19T21:01:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-19T21:01:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:01:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:01:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:32.286 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:01:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-19T21:01:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-19T21:01:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:01:32.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-19T21:01:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-19T21:01:32.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-19T21:01:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:01:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:01:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24319], [object_number=22, tag_tid=1, entry_tid=24318]]], state=connected] 2026-02-19T21:01:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.status 2026-02-19T21:01:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:01:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:01:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2026-02-19T21:01:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:01:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:01:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:01:33.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2026-02-19T21:01:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_44': 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 121ce6ddb101 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.121ce6ddb101 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:50:28 2026 2026-02-19T21:01:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:50:28 2026 2026-02-19T21:01:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:50:28 2026 2026-02-19T21:01:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:01:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 121ce6ddb101 2026-02-19T21:01:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:01:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:01:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: e27c2c16-d2aa-4d7a-8278-c5f7e2a947cd 2026-02-19T21:01:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:01:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_44 snap 2026-02-19T21:01:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:01:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:01:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.export 2026-02-19T21:01:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_44.export 2026-02-19T21:01:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_44.export 2026-02-19T21:01:33.354 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.export 2026-02-19T21:01:33.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:01:33.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_44.export 2026-02-19T21:01:33.686 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:01:33.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_44.export 2026-02-19T21:01:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_44.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_44.export 2026-02-19T21:01:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:01:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_45 2026-02-19T21:01:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_45 snap 2026-02-19T21:01:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:01:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2026-02-19T21:01:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2026-02-19T21:01:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:01:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:01:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:01:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2026-02-19T21:01:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:01:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:01:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:01:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2026-02-19T21:01:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:01:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:01:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:01:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:01:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:01:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:01:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2026-02-19T21:01:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:37.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:37.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:01:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:01:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2026-02-19T21:01:37.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-19T21:01:37.565 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:01:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:37.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:37.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_45 2026-02-19T21:01:37.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:01:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:01:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_45 2026-02-19T21:01:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:01:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:01:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:01:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:01:37.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:01:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:01:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:01:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_45 2026-02-19T21:01:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_45 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:01:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:01:37.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:01:37.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:37.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:01:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_45 2026-02-19T21:01:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_45 ']' 2026-02-19T21:01:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-19T21:01:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-19T21:01:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-19T21:01:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:37.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:37.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:37.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:37.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:37.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:37.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:37.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:37.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:37.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:37.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:37.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:37.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:37.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:37.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:37.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-19T21:01:37.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-19T21:01:37.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:37.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-19T21:01:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:37.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-19T21:01:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-19T21:01:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-19T21:01:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-19T21:01:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-19T21:01:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-19T21:01:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:01:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:01:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374], [object_number=45, tag_tid=1, entry_tid=48373], [object_number=44, tag_tid=1, entry_tid=48372]]], state=connected] 2026-02-19T21:01:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:01:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-19T21:01:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-19T21:01:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-19T21:01:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-19T21:01:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-19T21:01:38.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:01:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:01:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374], [object_number=45, tag_tid=1, entry_tid=48373], [object_number=44, tag_tid=1, entry_tid=48372]]], state=connected] 2026-02-19T21:01:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=48375' '!=' '' 2026-02-19T21:01:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:01:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=48375' = '' 2026-02-19T21:01:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:01:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:01:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:01:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:01:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:01:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:01:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:01:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:01:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:38.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:01:38.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=48375 2026-02-19T21:01:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 48375 2026-02-19T21:01:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:01:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:01:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_45 2026-02-19T21:01:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_45 ']' 2026-02-19T21:01:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-19T21:01:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-19T21:01:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-19T21:01:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:38.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:38.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:38.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:38.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-19T21:01:38.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-19T21:01:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-19T21:01:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:38.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-19T21:01:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-19T21:01:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-19T21:01:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:38.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-19T21:01:38.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-19T21:01:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-19T21:01:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:01:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:01:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374], [object_number=45, tag_tid=1, entry_tid=48373], [object_number=44, tag_tid=1, entry_tid=48372]]], state=connected] 2026-02-19T21:01:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:01:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-19T21:01:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-19T21:01:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-19T21:01:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-19T21:01:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-19T21:01:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:01:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:01:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:38.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374], [object_number=45, tag_tid=1, entry_tid=48373], [object_number=44, tag_tid=1, entry_tid=48372]]], state=connected] 2026-02-19T21:01:38.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=48375' '!=' 'object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_45 2026-02-19T21:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_45 ']' 2026-02-19T21:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-19T21:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-19T21:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:38.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-19T21:01:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-19T21:01:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-19T21:01:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-19T21:01:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-19T21:01:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-19T21:01:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-19T21:01:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-19T21:01:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-19T21:01:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:01:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:01:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374], [object_number=45, tag_tid=1, entry_tid=48373], [object_number=44, tag_tid=1, entry_tid=48372]]], state=connected] 2026-02-19T21:01:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:01:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-19T21:01:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-19T21:01:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-19T21:01:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-19T21:01:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:39.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-19T21:01:39.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:39.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:01:39.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:01:39.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:39.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:39.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374], [object_number=45, tag_tid=1, entry_tid=48373], [object_number=44, tag_tid=1, entry_tid=48372]]], state=connected] 2026-02-19T21:01:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=48375' '!=' 'object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:01:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_45 2026-02-19T21:01:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_45 ']' 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-19T21:01:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-19T21:01:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-19T21:01:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-19T21:01:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:40.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:40.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:40.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:40.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-19T21:01:40.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:40.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-19T21:01:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-19T21:01:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-19T21:01:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-19T21:01:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-19T21:01:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:01:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:01:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374], [object_number=45, tag_tid=1, entry_tid=48373], [object_number=44, tag_tid=1, entry_tid=48372]]], state=connected] 2026-02-19T21:01:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:01:40.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-19T21:01:40.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:40.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:40.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-19T21:01:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-19T21:01:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-19T21:01:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-19T21:01:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:01:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:01:40.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374], [object_number=45, tag_tid=1, entry_tid=48373], [object_number=44, tag_tid=1, entry_tid=48372]]], state=connected] 2026-02-19T21:01:40.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:40.208 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:40.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:40.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=47, tag_tid=1, entry_tid=48375' '!=' 'object_number=47, tag_tid=1, entry_tid=48375' 2026-02-19T21:01:40.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:40.209 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:01:41.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_45 2026-02-19T21:01:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_45 ']' 2026-02-19T21:01:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-19T21:01:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-19T21:01:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:41.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:41.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:41.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:41.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-19T21:01:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:41.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:41.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:41.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:41.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-19T21:01:41.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-19T21:01:41.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:41.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:41.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:41.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-19T21:01:41.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:41.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:41.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:41.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:41.883 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:41.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:41.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:41.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:41.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:41.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:41.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:41.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:41.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-19T21:01:41.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-19T21:01:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-19T21:01:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-19T21:01:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-19T21:01:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-19T21:01:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:01:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:01:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:41.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:41.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:41.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:01:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-19T21:01:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-19T21:01:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:01:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-19T21:01:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-19T21:01:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-19T21:01:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:01:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:01:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=48375], [object_number=46, tag_tid=1, entry_tid=48374]]], state=connected] 2026-02-19T21:01:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.status 2026-02-19T21:01:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:01:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:01:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2026-02-19T21:01:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:01:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:01:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:01:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2026-02-19T21:01:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_45': 2026-02-19T21:01:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:01:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:01:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:01:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a639d4d9c8 2026-02-19T21:01:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a639d4d9c8 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:50:37 2026 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:50:37 2026 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:51:38 2026 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a639d4d9c8 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:01:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:01:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: ea02bcb2-4a9c-449e-beda-ecd26614ee2d 2026-02-19T21:01:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:01:43.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:43.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_45 snap 2026-02-19T21:01:43.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:43.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:01:43.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:43.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:01:43.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.export 2026-02-19T21:01:43.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_45.export 2026-02-19T21:01:43.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_45.export 2026-02-19T21:01:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.export 2026-02-19T21:01:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:01:43.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_45.export 2026-02-19T21:01:43.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:01:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_45.export 2026-02-19T21:01:43.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_45.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_45.export 2026-02-19T21:01:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:01:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_46 2026-02-19T21:01:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_46 snap 2026-02-19T21:01:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:01:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:01:44.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2026-02-19T21:01:44.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2026-02-19T21:01:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:01:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:01:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:01:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2026-02-19T21:01:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:01:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:01:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:01:45.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:01:45.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2026-02-19T21:01:45.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:01:45.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:45.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:45.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:01:45.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:01:45.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:45.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:01:45.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:45.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:01:45.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:45.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-19T21:01:45.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:01:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:01:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:01:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:01:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:01:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:01:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-19T21:01:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:01:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:01:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:01:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:01:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:01:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:01:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:01:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_46 2026-02-19T21:01:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:01:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:01:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:01:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:01:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_46 2026-02-19T21:01:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:01:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:01:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:01:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:01:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:01:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:01:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:01:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:01:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_46 2026-02-19T21:01:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_46 ']' 2026-02-19T21:01:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-19T21:01:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-19T21:01:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:01:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-19T21:01:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-19T21:01:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:01:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:01:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:01:46.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:01:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:01:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:01:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-19T21:01:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-19T21:01:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:01:46.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:01:46.302 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:46.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:01:46.303 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:01:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:01:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:46.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36990' '!=' '' 2026-02-19T21:01:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:01:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36990' = '' 2026-02-19T21:01:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:01:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:01:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:01:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:01:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:01:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:01:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:01:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:46.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:01:46.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:46.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:01:46.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:01:46.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=36990 2026-02-19T21:01:46.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 36990 2026-02-19T21:01:46.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:01:46.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:46.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:01:46.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_46 2026-02-19T21:01:46.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_46 ']' 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:01:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:46.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:46.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:46.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:46.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:46.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:46.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:46.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-19T21:01:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-19T21:01:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:01:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:01:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:01:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:01:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:01:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:46.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:01:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-19T21:01:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-19T21:01:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:01:46.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:01:46.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:46.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:01:46.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:01:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:01:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:46.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:46.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:46.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:46.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36990' '!=' 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:01:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_46 2026-02-19T21:01:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_46 ']' 2026-02-19T21:01:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-19T21:01:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-19T21:01:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:01:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-19T21:01:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-19T21:01:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:01:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:01:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:01:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:01:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:01:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:47.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:47.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:01:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-19T21:01:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-19T21:01:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:01:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:01:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:47.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:47.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:01:47.232 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:47.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:01:47.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:01:47.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:47.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:47.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:47.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:47.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:47.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36990' '!=' 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:47.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:47.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:01:48.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_46 2026-02-19T21:01:48.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:48.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:48.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:48.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:48.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_46 ']' 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:01:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:01:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-19T21:01:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-19T21:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:01:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:48.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:01:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:01:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:01:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-19T21:01:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-19T21:01:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:01:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:01:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:01:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:01:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:01:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:01:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:01:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:01:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:48.200 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T21:01:48.200 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T21:01:48.200 INFO:tasks.rbd_mirror_thrash:waiting for 6.121676345777224 secs before reviving daemons 2026-02-19T21:01:48.200 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:01:48.189+0000 7fbf1e429640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 95028) UID: 1000 2026-02-19T21:01:48.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:01:48.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:01:48.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:01:48.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:48.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:01:48.229 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:01:48.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:48.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36990' '!=' 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:01:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:01:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:01:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_46 2026-02-19T21:01:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:01:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:01:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:01:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_46 ']' 2026-02-19T21:01:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-19T21:01:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-19T21:01:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:01:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:01:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:01:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:01:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:01:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:01:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:49.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:49.849 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:49.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:49.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:49.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:49.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:49.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:49.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:49.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:49.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:49.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:49.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-19T21:01:49.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:49.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:49.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:49.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:49.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:49.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:49.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:50.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:50.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_46 2026-02-19T21:01:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:01:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:01:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:01:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:50.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:50.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:50.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:50.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:01:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:51.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:51.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:51.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-19T21:01:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:51.270 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:51.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:51.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:51.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_46 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:01:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:01:51.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:51.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:51.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:01:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:01:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:53.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:53.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:53.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:53.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-19T21:01:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-19T21:01:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:53.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-19T21:01:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:53.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:53.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:53.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:53.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:53.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:53.602 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:53.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:53.605 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_46 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:01:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:01:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:01:53.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:53.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:53.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:53.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:53.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:53.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 4 2026-02-19T21:01:54.322 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T21:01:54.323 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T21:01:54.323 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T21:01:54.323 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T21:01:54.323 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T21:01:54.325 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T21:01:54.325 INFO:tasks.rbd_mirror_thrash:waiting for 99.57935094060522 secs before thrashing 2026-02-19T21:01:57.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:01:57.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:01:57.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:57.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:01:57.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:01:57.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:57.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:57.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:01:57.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:01:57.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:57.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:57.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:01:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:01:57.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:57.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:57.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:01:57.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:57.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:57.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:57.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:01:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:01:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:01:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:01:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:01:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:01:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:01:57.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-19T21:01:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:01:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:01:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:01:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:01:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:01:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:57.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:57.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:01:57.831 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:57.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:57.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:57.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_46 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:01:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:01:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:01:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_46 2026-02-19T21:01:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:01:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:01:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:01:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:01:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 8 2026-02-19T21:02:01.153 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T21:02:01.141+0000 7f5ce3652640 -1 reset not still connected to 0x56103bb5d450 2026-02-19T21:02:01.154 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T21:02:01.145+0000 7f5ce3652640 -1 reset not still connected to 0x56103d368ea0 2026-02-19T21:02:01.154 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T21:02:01.145+0000 7f5ce3652640 -1 reset not still connected to 0x56103d369ee0 2026-02-19T21:02:05.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:05.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:02:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:05.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:05.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:06.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-19T21:02:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:06.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_46 2026-02-19T21:02:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:02:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:02:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:02:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:02:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:02:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:02:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:02:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:02:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 8 2026-02-19T21:02:14.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:14.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:02:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:14.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:14.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:14.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-19T21:02:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:14.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:14.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:14.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:14.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:14.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:14.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:14.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:14.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:14.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:14.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-19T21:02:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:14.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:14.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:14.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_46 2026-02-19T21:02:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:02:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:02:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:02:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:02:14.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:02:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:02:14.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:02:14.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:02:14.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:14.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:14.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 8 2026-02-19T21:02:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:02:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:22.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:22.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:22.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:22.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-19T21:02:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:22.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:22.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:22.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_46 2026-02-19T21:02:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:02:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:3 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=3 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=3 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-19T21:02:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:3 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:3 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=3 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=3 2026-02-19T21:02:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=3 2026-02-19T21:02:22.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-19T21:02:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:02:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:02:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-19T21:02:22.766 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-19T21:02:22.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:22.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:22.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:22.821 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 8 2026-02-19T21:02:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:30.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:02:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:30.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:30.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:30.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:30.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:30.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:02:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:02:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:02:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:02:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:30.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:30.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:02:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-19T21:02:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-19T21:02:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:02:30.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:02:30.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:30.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:30.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:02:30.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:30.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:02:30.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:02:30.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:30.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:30.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:02:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:02:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36990' '!=' 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:02:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:02:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_46 2026-02-19T21:02:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:02:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_46 ']' 2026-02-19T21:02:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-19T21:02:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-19T21:02:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:02:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-19T21:02:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-19T21:02:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:02:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:02:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:33.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:33.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:02:33.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:02:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:02:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:33.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:02:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-19T21:02:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-19T21:02:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:02:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:02:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:02:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:02:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:02:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=36990], [object_number=33, tag_tid=1, entry_tid=36989], [object_number=32, tag_tid=1, entry_tid=36988], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:02:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:02:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=36990' '!=' 'object_number=34, tag_tid=1, entry_tid=36990' 2026-02-19T21:02:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:02:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_46 2026-02-19T21:02:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:02:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_46 ']' 2026-02-19T21:02:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-19T21:02:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-19T21:02:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-19T21:02:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:35.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-19T21:02:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-19T21:02:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-19T21:02:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:02:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:02:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:35.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:35.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:02:35.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:02:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:02:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:35.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:02:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-19T21:02:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-19T21:02:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:02:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-19T21:02:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-19T21:02:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-19T21:02:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:35.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 8 2026-02-19T21:02:35.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 8 2026-02-19T21:02:35.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:35.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:35.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=36990], [object_number=35, tag_tid=1, entry_tid=36987]]], state=connected] 2026-02-19T21:02:35.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.status 2026-02-19T21:02:35.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:02:35.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-19T21:02:35.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:35.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2026-02-19T21:02:35.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:35.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:35.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:02:35.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:02:35.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:02:35.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:02:35.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:02:36.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2026-02-19T21:02:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_46': 2026-02-19T21:02:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:02:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11fcb48699cf 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11fcb48699cf 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:50:50 2026 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:50:50 2026 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:51:51 2026 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:02:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11fcb48699cf 2026-02-19T21:02:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:02:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:02:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 034c8394-8f9f-4a3f-8f42-f58610f376f5 2026-02-19T21:02:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:02:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_46 snap 2026-02-19T21:02:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:02:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:02:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:02:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.export 2026-02-19T21:02:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_46.export 2026-02-19T21:02:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_46.export 2026-02-19T21:02:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.export 2026-02-19T21:02:36.688 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:02:36.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_46.export 2026-02-19T21:02:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:02:36.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_46.export 2026-02-19T21:02:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_46.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_46.export 2026-02-19T21:02:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:02:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_47 2026-02-19T21:02:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_47 snap 2026-02-19T21:02:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:02:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:02:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:02:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2026-02-19T21:02:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2026-02-19T21:02:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:02:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:02:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:02:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2026-02-19T21:02:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:02:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:02:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:02:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2026-02-19T21:02:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:02:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:02:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:02:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:02:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:02:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:02:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2026-02-19T21:02:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:02:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:02:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:02:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:02:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:02:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:02:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:02:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:02:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:02:39.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:39.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:39.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:39.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:02:39.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:02:39.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:02:39.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:02:39.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2026-02-19T21:02:40.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:02:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_47 2026-02-19T21:02:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:02:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:02:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:02:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:02:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:02:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:02:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:02:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:02:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:40.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_47 2026-02-19T21:02:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:02:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:40.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:40.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_47 2026-02-19T21:02:40.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T21:02:40.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:40.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:02:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:02:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:02:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:02:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T21:02:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T21:02:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_47 2026-02-19T21:02:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:02:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:02:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:02:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:02:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:02:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:02:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:02:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_47 2026-02-19T21:02:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:02:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:02:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_47 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:02:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:02:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:02:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:02:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_47 2026-02-19T21:02:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_47 ']' 2026-02-19T21:02:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-19T21:02:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-19T21:02:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-19T21:02:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:40.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:40.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:40.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:40.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:40.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:40.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-19T21:02:40.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:40.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:40.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:40.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:40.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:40.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:40.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:40.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-19T21:02:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-19T21:02:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:40.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:40.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:40.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23358], [object_number=21, tag_tid=1, entry_tid=23357], [object_number=20, tag_tid=1, entry_tid=23356], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:40.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:40.640 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:02:40.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-19T21:02:40.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:40.641 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-19T21:02:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:40.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:40.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:40.643 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:40.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:40.644 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:40.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23358], [object_number=21, tag_tid=1, entry_tid=23357], [object_number=20, tag_tid=1, entry_tid=23356], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:40.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23358' '!=' '' 2026-02-19T21:02:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:02:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23358' = '' 2026-02-19T21:02:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:02:40.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:02:40.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:02:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:02:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:40.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:02:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:02:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:02:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:40.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:02:40.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:40.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:02:40.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=23358 2026-02-19T21:02:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 23358 2026-02-19T21:02:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:02:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:02:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_47 2026-02-19T21:02:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_47 ']' 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:41.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:41.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:41.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:41.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:41.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:41.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:41.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:41.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:41.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:41.249 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-19T21:02:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-19T21:02:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:41.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:41.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:41.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:41.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:41.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23358], [object_number=21, tag_tid=1, entry_tid=23357], [object_number=20, tag_tid=1, entry_tid=23356], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:02:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-19T21:02:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-19T21:02:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:41.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:41.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:41.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:41.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:41.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23358], [object_number=21, tag_tid=1, entry_tid=23357], [object_number=20, tag_tid=1, entry_tid=23356], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23358' '!=' 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_47 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_47 ']' 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:41.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:41.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:41.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:41.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:41.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:41.854 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:41.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:41.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:41.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:41.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:41.855 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:41.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:41.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:41.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:41.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:42.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:42.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:42.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:42.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:42.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:42.012 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23358], [object_number=21, tag_tid=1, entry_tid=23357], [object_number=20, tag_tid=1, entry_tid=23356], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:02:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-19T21:02:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-19T21:02:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23358], [object_number=21, tag_tid=1, entry_tid=23357], [object_number=20, tag_tid=1, entry_tid=23356], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23358' '!=' 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:02:42.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_47 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_47 ']' 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:42.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:42.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:42.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:42.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:42.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:42.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-19T21:02:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:42.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:42.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:43.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:43.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-19T21:02:43.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:43.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:43.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-19T21:02:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-19T21:02:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:43.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:43.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:43.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23358], [object_number=21, tag_tid=1, entry_tid=23357], [object_number=20, tag_tid=1, entry_tid=23356], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:02:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-19T21:02:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-19T21:02:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23358], [object_number=21, tag_tid=1, entry_tid=23357], [object_number=20, tag_tid=1, entry_tid=23356], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23358' '!=' 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:02:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_47 2026-02-19T21:02:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_47 ']' 2026-02-19T21:02:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-19T21:02:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-19T21:02:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-19T21:02:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:44.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:44.803 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-19T21:02:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:44.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:44.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:44.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:44.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:44.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:44.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:44.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-19T21:02:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-19T21:02:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:45.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23358], [object_number=21, tag_tid=1, entry_tid=23357], [object_number=20, tag_tid=1, entry_tid=23356], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:45.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:45.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:02:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-19T21:02:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-19T21:02:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:45.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:45.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23358], [object_number=21, tag_tid=1, entry_tid=23357], [object_number=20, tag_tid=1, entry_tid=23356], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:45.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23358' '!=' 'object_number=22, tag_tid=1, entry_tid=23358' 2026-02-19T21:02:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:02:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_47 2026-02-19T21:02:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_47 ']' 2026-02-19T21:02:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-19T21:02:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-19T21:02:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:47.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:47.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:47.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:47.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:47.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:47.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:47.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:47.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:47.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:47.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:47.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-19T21:02:47.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:47.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:47.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:47.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:47.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-19T21:02:47.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:47.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:47.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:47.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-19T21:02:47.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:47.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:47.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:47.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:47.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:47.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-19T21:02:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-19T21:02:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-19T21:02:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:47.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:47.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:02:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-19T21:02:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-19T21:02:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:02:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-19T21:02:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-19T21:02:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:47.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:47.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-19T21:02:47.376 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 5 2026-02-19T21:02:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 5 2026-02-19T21:02:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23358], [object_number=23, tag_tid=1, entry_tid=23355]]], state=connected] 2026-02-19T21:02:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.status 2026-02-19T21:02:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:02:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-19T21:02:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2026-02-19T21:02:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:02:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:02:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:02:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:02:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_47': 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 120c8f28a130 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.120c8f28a130 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:51:02 2026 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:51:02 2026 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:52:03 2026 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 120c8f28a130 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 62ca484a-4a62-4982-96ae-1512aa84da3e 2026-02-19T21:02:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:02:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_47 snap 2026-02-19T21:02:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:02:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:02:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:02:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.export 2026-02-19T21:02:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_47.export 2026-02-19T21:02:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_47.export 2026-02-19T21:02:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.export 2026-02-19T21:02:48.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:02:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_47.export 2026-02-19T21:02:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:02:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_47.export 2026-02-19T21:02:48.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_47.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_47.export 2026-02-19T21:02:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:02:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_48 2026-02-19T21:02:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_48 snap 2026-02-19T21:02:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:02:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:02:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:02:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:02:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2026-02-19T21:02:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:02:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2026-02-19T21:02:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:02:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:02:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:02:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:02:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2026-02-19T21:02:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:02:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:02:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:02:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:02:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2026-02-19T21:02:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:02:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:02:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:02:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:02:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:02:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:02:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2026-02-19T21:02:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:02:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:02:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:02:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:02:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:02:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:02:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:02:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:02:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:02:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:02:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:02:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:02:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:02:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2026-02-19T21:02:53.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-19T21:02:53.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:02:53.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_48 2026-02-19T21:02:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:02:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:02:53.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:02:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:02:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:53.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:53.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:53.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:53.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_48 2026-02-19T21:02:53.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-19T21:02:53.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:02:53.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_48 2026-02-19T21:02:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:2 2026-02-19T21:02:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:02:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:2 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:2 2026-02-19T21:02:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:02:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:02:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:02:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:02:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=2 2026-02-19T21:02:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=2 2026-02-19T21:02:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:53.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:53.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_48 2026-02-19T21:02:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:02:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:02:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:02:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:02:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:02:53.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:02:53.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:02:53.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_48 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_48 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:02:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:02:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:02:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:02:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:02:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:02:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:02:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:02:53.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_48 2026-02-19T21:02:53.388 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:02:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_48 ']' 2026-02-19T21:02:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-19T21:02:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-19T21:02:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-19T21:02:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-19T21:02:53.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:53.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-19T21:02:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-19T21:02:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-19T21:02:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-19T21:02:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:53.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:53.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:53.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-19T21:02:53.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-19T21:02:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_48 2026-02-19T21:02:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-19T21:02:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_48 2026-02-19T21:02:53.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:53.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-19T21:02:53.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-19T21:02:53.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:53.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:53.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-19T21:02:53.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:02:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:02:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49243], [object_number=46, tag_tid=1, entry_tid=49242]]], state=connected] 2026-02-19T21:02:53.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49244], [object_number=47, tag_tid=1, entry_tid=49243], [object_number=46, tag_tid=1, entry_tid=49242], [object_number=45, tag_tid=1, entry_tid=49241]]], state=connected] 2026-02-19T21:02:53.653 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:53.655 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:02:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-19T21:02:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_48 2026-02-19T21:02:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-19T21:02:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_48 2026-02-19T21:02:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:53.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-19T21:02:53.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-19T21:02:53.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:53.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:53.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-19T21:02:53.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:53.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:02:53.692 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:02:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49243], [object_number=46, tag_tid=1, entry_tid=49242]]], state=connected] 2026-02-19T21:02:53.693 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=44, tag_tid=1, entry_tid=49244], [object_number=47, tag_tid=1, entry_tid=49243], [object_number=46, tag_tid=1, entry_tid=49242], [object_number=45, tag_tid=1, entry_tid=49241]]], state=connected] 2026-02-19T21:02:53.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=44, tag_tid=1, entry_tid=49244' 2026-02-19T21:02:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=44, tag_tid=1, entry_tid=49244' 2026-02-19T21:02:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=49244' '!=' '' 2026-02-19T21:02:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:02:53.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=44, tag_tid=1, entry_tid=49244' = '' 2026-02-19T21:02:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=44, tag_tid=1, entry_tid=49244' 2026-02-19T21:02:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:02:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:02:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:53.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:02:53.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=44, tag_tid=1, entry_tid=49244' 2026-02-19T21:02:53.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:02:53.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:53.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:02:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:02:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:02:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:02:53.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=44, tag_tid=1, entry_tid=49244' 2026-02-19T21:02:53.705 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:02:53.706 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=49244 2026-02-19T21:02:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 49244 2026-02-19T21:02:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:02:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:02:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_48 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_48 ']' 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-19T21:02:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-19T21:02:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-19T21:02:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-19T21:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-19T21:02:54.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-19T21:02:54.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:54.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:54.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:2 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=2 2026-02-19T21:02:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=2 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:2 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:2 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=2 2026-02-19T21:02:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-19T21:02:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=2 2026-02-19T21:02:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=2 2026-02-19T21:02:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-19T21:02:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-19T21:02:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-19T21:02:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-19T21:02:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_48 2026-02-19T21:02:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-19T21:02:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_48 2026-02-19T21:02:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-19T21:02:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-19T21:02:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-19T21:02:54.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:02:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:02:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49243], [object_number=46, tag_tid=1, entry_tid=49242]]], state=connected] 2026-02-19T21:02:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49243], [object_number=46, tag_tid=1, entry_tid=49242]]], state=connected] 2026-02-19T21:02:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:02:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-19T21:02:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_48 2026-02-19T21:02:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-19T21:02:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_48 2026-02-19T21:02:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-19T21:02:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-19T21:02:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-19T21:02:54.182 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 11 2026-02-19T21:02:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 11 2026-02-19T21:02:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49243], [object_number=46, tag_tid=1, entry_tid=49242]]], state=connected] 2026-02-19T21:02:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=49243], [object_number=46, tag_tid=1, entry_tid=49242]]], state=connected] 2026-02-19T21:02:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.status 2026-02-19T21:02:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:02:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-19T21:02:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2026-02-19T21:02:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:02:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:02:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:02:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:02:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:02:55.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2026-02-19T21:02:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_48': 2026-02-19T21:02:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:02:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:02:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:02:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 121c12b2aae9 2026-02-19T21:02:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.121c12b2aae9 2026-02-19T21:02:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:02:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:02:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:02:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:02:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:51:09 2026 2026-02-19T21:02:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:51:09 2026 2026-02-19T21:02:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:52:10 2026 2026-02-19T21:02:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:02:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 121c12b2aae9 2026-02-19T21:02:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:02:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:02:55.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: d8a1d799-e7a6-4a50-82af-234321276cf5 2026-02-19T21:02:55.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:02:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_48 snap 2026-02-19T21:02:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:02:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:02:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:02:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.export 2026-02-19T21:02:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_48.export 2026-02-19T21:02:55.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_48.export 2026-02-19T21:02:55.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.export 2026-02-19T21:02:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:02:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_48.export 2026-02-19T21:02:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:02:55.791 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_48.export 2026-02-19T21:02:55.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_48.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_48.export 2026-02-19T21:02:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:02:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_49 2026-02-19T21:02:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_49 snap 2026-02-19T21:02:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:02:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:02:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:02:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:02:56.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2026-02-19T21:02:56.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:56.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:56.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:02:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2026-02-19T21:02:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:02:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:02:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:02:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:02:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2026-02-19T21:02:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:02:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:02:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:02:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:02:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2026-02-19T21:02:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:02:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:02:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:02:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:02:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:02:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:02:57.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:02:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:02:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:02:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:02:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:02:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:02:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:02:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2026-02-19T21:02:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-19T21:02:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:02:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2026-02-19T21:02:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:02:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:02:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:02:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:02:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:02:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:02:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:02:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:58.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2026-02-19T21:02:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:02:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:02:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:02:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:02:58.115 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:02:58.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:02:58.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:02:58.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:58.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_49 2026-02-19T21:02:58.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:02:58.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_49 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:02:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:02:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:02:58.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_49 2026-02-19T21:02:58.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_49 ']' 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:58.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:58.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-19T21:02:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-19T21:02:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-19T21:02:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-19T21:02:58.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:58.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-19T21:02:58.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-19T21:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:02:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:02:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:02:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:02:58.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14004], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002], [object_number=13, tag_tid=1, entry_tid=14001]]], state=connected] 2026-02-19T21:02:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-19T21:02:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-19T21:02:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-19T21:02:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:02:58.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:02:58.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:58.516 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:02:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:02:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:02:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:02:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14004], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002], [object_number=13, tag_tid=1, entry_tid=14001]]], state=connected] 2026-02-19T21:02:58.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:58.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:58.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=12, tag_tid=1, entry_tid=14004' '!=' '' 2026-02-19T21:02:58.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:02:58.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=12, tag_tid=1, entry_tid=14004' = '' 2026-02-19T21:02:58.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:58.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=13, tag_tid=3, entry_tid=1' 2026-02-19T21:02:58.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:02:58.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:58.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=3 2026-02-19T21:02:58.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:58.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:02:58.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:58.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:02:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=13, tag_tid=3, entry_tid=1' 2026-02-19T21:02:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:02:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:02:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:02:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:02:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=14004 2026-02-19T21:02:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 3 = 1 -a 1 -le 14004 2026-02-19T21:02:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:02:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_49 2026-02-19T21:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_49 ']' 2026-02-19T21:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-19T21:02:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-19T21:02:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-19T21:02:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:58.773 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-19T21:02:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-19T21:02:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-19T21:02:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:58.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:58.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:58.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:58.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:58.848 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-19T21:02:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-19T21:02:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-19T21:02:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:02:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:02:58.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:58.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:02:58.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:02:58.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:02:58.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:58.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:02:58.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14004], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002], [object_number=13, tag_tid=1, entry_tid=14001]]], state=connected] 2026-02-19T21:02:58.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-19T21:02:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-19T21:02:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-19T21:02:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:02:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:02:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:02:58.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:02:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:02:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:02:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14004], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002], [object_number=13, tag_tid=1, entry_tid=14001]]], state=connected] 2026-02-19T21:02:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=12, tag_tid=1, entry_tid=14004' '!=' 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.4 2026-02-19T21:02:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_49 2026-02-19T21:02:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:02:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:02:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:02:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_49 ']' 2026-02-19T21:02:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-19T21:02:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-19T21:02:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:02:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:02:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:02:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:02:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:02:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:02:59.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:02:59.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:02:59.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:59.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-19T21:02:59.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:02:59.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:02:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:02:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:02:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:02:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:02:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:02:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:02:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:02:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:02:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-19T21:02:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-19T21:02:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:02:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:02:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:02:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-19T21:02:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:02:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:02:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:02:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:02:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:02:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:02:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:02:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:02:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-19T21:02:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:02:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-19T21:02:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-19T21:02:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:02:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:02:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:02:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:02:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:59.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:02:59.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:02:59.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:59.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:02:59.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14004], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002], [object_number=13, tag_tid=1, entry_tid=14001]]], state=connected] 2026-02-19T21:02:59.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-19T21:02:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-19T21:02:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-19T21:02:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:02:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:02:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:02:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:02:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:02:59.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:02:59.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:02:59.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:59.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:02:59.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:59.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:02:59.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:02:59.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:02:59.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:02:59.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14004], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002], [object_number=13, tag_tid=1, entry_tid=14001]]], state=connected] 2026-02-19T21:02:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:02:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=12, tag_tid=1, entry_tid=14004' '!=' 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:02:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:02:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.8 2026-02-19T21:03:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_49 2026-02-19T21:03:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_49 ']' 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:03:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-19T21:03:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:03:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:03:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:03:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:03:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:03:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:03:00.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:00.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:00.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:00.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-19T21:03:00.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-19T21:03:00.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:03:00.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:03:00.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:00.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:03:00.503 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:03:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:03:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:03:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:03:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:03:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-19T21:03:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-19T21:03:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-19T21:03:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:03:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:03:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:03:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:03:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:03:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:03:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:03:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:03:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:03:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14004], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002], [object_number=13, tag_tid=1, entry_tid=14001]]], state=connected] 2026-02-19T21:03:00.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:00.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-19T21:03:00.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-19T21:03:00.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:00.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:00.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:00.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-19T21:03:00.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:00.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:00.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:00.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:03:00.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:03:00.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:03:00.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:03:00.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:00.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:03:00.617 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:03:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:03:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:03:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:03:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14004], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002], [object_number=13, tag_tid=1, entry_tid=14001]]], state=connected] 2026-02-19T21:03:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:00.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:03:00.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:03:00.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=12, tag_tid=1, entry_tid=14004' '!=' 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:03:00.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:03:00.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1.6 2026-02-19T21:03:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_49 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_49 ']' 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:03:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:03:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:03:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:03:02.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:03:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:03:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:03:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:03:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:03:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:03:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:03:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:03:02.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-19T21:03:02.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-19T21:03:02.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:03:02.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:03:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-19T21:03:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:03:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:03:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:03:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:03:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:03:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:03:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:03:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:03:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:03:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:03:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:03:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-19T21:03:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-19T21:03:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-19T21:03:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:02.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:02.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:02.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:03:02.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:03:02.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:03:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:03:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:03:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:03:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14004], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002], [object_number=13, tag_tid=1, entry_tid=14001]]], state=connected] 2026-02-19T21:03:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-19T21:03:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-19T21:03:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-19T21:03:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:03:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:03:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:03:02.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:03:02.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:02.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:02.441 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:03:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:03:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14004], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002], [object_number=13, tag_tid=1, entry_tid=14001]]], state=connected] 2026-02-19T21:03:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:03:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:03:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=12, tag_tid=1, entry_tid=14004' '!=' 'object_number=12, tag_tid=1, entry_tid=14004' 2026-02-19T21:03:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:03:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 2 2026-02-19T21:03:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_49 2026-02-19T21:03:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:03:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:03:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_49 ']' 2026-02-19T21:03:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-19T21:03:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-19T21:03:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:03:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-19T21:03:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-19T21:03:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:03:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:03:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-19T21:03:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:03:04.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:03:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:03:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:03:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:03:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:03:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:03:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:03:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:03:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:03:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:03:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:03:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:03:04.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:03:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-19T21:03:04.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-19T21:03:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-19T21:03:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:03:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:03:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:03:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:03:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:03:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:04.865 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:03:04.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:03:04.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:03:04.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:03:04.866 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:03:04.869 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:04.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-19T21:03:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-19T21:03:04.872 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-19T21:03:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:03:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:03:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-19T21:03:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-19T21:03:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:03:04.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:04.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-19T21:03:04.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 3 2026-02-19T21:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 3 2026-02-19T21:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14003], [object_number=14, tag_tid=1, entry_tid=14002]]], state=connected] 2026-02-19T21:03:05.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.status 2026-02-19T21:03:05.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-19T21:03:05.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=3, entry_tid=1' 2026-02-19T21:03:05.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2026-02-19T21:03:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:03:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:03:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:03:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:03:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:03:06.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2026-02-19T21:03:06.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_49': 2026-02-19T21:03:06.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:03:06.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:03:06.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:03:06.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 10a677bbdcd8 2026-02-19T21:03:06.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.10a677bbdcd8 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:51:23 2026 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:51:23 2026 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:00:51 2026 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 10a677bbdcd8 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:03:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:03:06.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 0a39469c-5889-44f4-9724-5da19a7298c8 2026-02-19T21:03:06.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:03:06.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:06.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_49 snap 2026-02-19T21:03:06.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:06.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:03:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:03:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:03:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.export 2026-02-19T21:03:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_49.export 2026-02-19T21:03:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_49.export 2026-02-19T21:03:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.export 2026-02-19T21:03:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:03:06.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_49.export 2026-02-19T21:03:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:03:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_49.export 2026-02-19T21:03:06.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_49.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_49.export 2026-02-19T21:03:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_50 2026-02-19T21:03:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_snap cluster2 mirror image_50 snap 2026-02-19T21:03:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:03:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:03:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2026-02-19T21:03:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2026-02-19T21:03:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:03:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2026-02-19T21:03:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:03:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=started 2026-02-19T21:03:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:03:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:03:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:03:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2026-02-19T21:03:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:03:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=started 2026-02-19T21:03:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local status_result 2026-02-19T21:03:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=stopped 2026-02-19T21:03:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2026-02-19T21:03:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -i state 2026-02-19T21:03:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_instance=cluster1 2026-02-19T21:03:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:03:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance=cluster1 2026-02-19T21:03:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local loop_instance 2026-02-19T21:03:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:03:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sleep 0 2026-02-19T21:03:10.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:03:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ seq 0 3 2026-02-19T21:03:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2026-02-19T21:03:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:0 2026-02-19T21:03:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:03:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:03:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:03:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:0 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:03:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:0 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=0 2026-02-19T21:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=0 2026-02-19T21:03:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:03:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2026-02-19T21:03:11.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-19T21:03:11.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror status : get status for rbd mirror 2026-02-19T21:03:11.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:03:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_50 2026-02-19T21:03:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1:1 2026-02-19T21:03:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ shift 2026-02-19T21:03:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=cluster 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval cluster=cluster1 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:03:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local local_cluster=cluster1:1 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster=cluster1 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local val=cluster1:1 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local cluster_var_name=local_cluster 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local instance_var_name=instance 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ cluster=cluster1 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval local_cluster=cluster1 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ local_cluster=cluster1 2026-02-19T21:03:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ eval instance=1 2026-02-19T21:03:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ instance=1 2026-02-19T21:03:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:03:11.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:11.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:11.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:11.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_50 2026-02-19T21:03:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ return 0 2026-02-19T21:03:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ status_result=' "state": "Replaying"' 2026-02-19T21:03:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo ' "state": "Replaying"' 2026-02-19T21:03:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ grep -i Replaying 2026-02-19T21:03:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: "state": "Replaying" 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=started 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test started = started 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_50 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' journal = journal ']' 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_50 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_cluster=cluster1 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local local_pool=mirror 2026-02-19T21:03:11.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local remote_pool=mirror 2026-02-19T21:03:11.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:03:11.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-19T21:03:11.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-19T21:03:11.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:03:11.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:03:11.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:03:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_50 2026-02-19T21:03:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:03:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:03:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_50 ']' 2026-02-19T21:03:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-19T21:03:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-19T21:03:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:03:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:03:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:03:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:03:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:03:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:03:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-19T21:03:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:03:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:03:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:03:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:03:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:03:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:03:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:03:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:03:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:03:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:03:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:03:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:03:11.329 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:03:11.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:11.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:11.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:11.339 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-19T21:03:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-19T21:03:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:03:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:03:11.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:11.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-19T21:03:11.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:03:11.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:11.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:03:11.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:03:11.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:03:11.401 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:03:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:03:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-19T21:03:11.469 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-19T21:03:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_50 2026-02-19T21:03:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-19T21:03:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_50 2026-02-19T21:03:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:03:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-19T21:03:11.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-19T21:03:11.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:03:11.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-19T21:03:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 13 2026-02-19T21:03:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 13 2026-02-19T21:03:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:03:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=55983], [object_number=54, tag_tid=1, entry_tid=55982]]], state=connected] 2026-02-19T21:03:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=55, tag_tid=1, entry_tid=55983], [object_number=54, tag_tid=1, entry_tid=55982], [object_number=53, tag_tid=1, entry_tid=55981], [object_number=52, tag_tid=1, entry_tid=55980]]], state=connected] 2026-02-19T21:03:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-19T21:03:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-19T21:03:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_50 2026-02-19T21:03:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-19T21:03:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_50 2026-02-19T21:03:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:03:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-19T21:03:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-19T21:03:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:03:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-19T21:03:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 13 2026-02-19T21:03:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 13 2026-02-19T21:03:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:03:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=55983], [object_number=54, tag_tid=1, entry_tid=55982]]], state=connected] 2026-02-19T21:03:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=55, tag_tid=1, entry_tid=55983], [object_number=54, tag_tid=1, entry_tid=55982], [object_number=53, tag_tid=1, entry_tid=55981], [object_number=52, tag_tid=1, entry_tid=55980]]], state=connected] 2026-02-19T21:03:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=55, tag_tid=1, entry_tid=55983' 2026-02-19T21:03:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=55, tag_tid=1, entry_tid=55983' 2026-02-19T21:03:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=55, tag_tid=1, entry_tid=55983' '!=' '' 2026-02-19T21:03:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:03:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 'object_number=55, tag_tid=1, entry_tid=55983' = '' 2026-02-19T21:03:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_mirror_pos='object_number=55, tag_tid=1, entry_tid=55983' 2026-02-19T21:03:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-19T21:03:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:03:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:03:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_tag=2 2026-02-19T21:03:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=55, tag_tid=1, entry_tid=55983' 2026-02-19T21:03:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-19T21:03:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:03:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_tag=1 2026-02-19T21:03:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-19T21:03:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:03:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:03:11.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_entry=1 2026-02-19T21:03:11.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo 'object_number=55, tag_tid=1, entry_tid=55983' 2026-02-19T21:03:11.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-19T21:03:11.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut -d= -f 2 2026-02-19T21:03:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_entry=55983 2026-02-19T21:03:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 = 1 -a 1 -le 55983 2026-02-19T21:03:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:03:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-19T21:03:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.2 2026-02-19T21:03:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ flush cluster1 mirror image_50 2026-02-19T21:03:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:03:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local 'cmd=rbd mirror flush' 2026-02-19T21:03:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n image_50 ']' 2026-02-19T21:03:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-19T21:03:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-19T21:03:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_instance=cluster1 2026-02-19T21:03:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance=cluster1 2026-02-19T21:03:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loop_instance 2026-02-19T21:03:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-19T21:03:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-19T21:03:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:0 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:03:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:0 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:0 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:03:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:03:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-19T21:03:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:03:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:03:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=0 2026-02-19T21:03:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=0 2026-02-19T21:03:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-19T21:03:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-19T21:03:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-19T21:03:11.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-19T21:03:11.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd mirror flush : flush rbd mirror 2026-02-19T21:03:11.822 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:admin_socket: invalid command 2026-02-19T21:03:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:03:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-19T21:03:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:03:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:03:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:03:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1:1 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=1 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=1 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-19T21:03:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local local_cluster=cluster1:1 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local val=cluster1:1 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster_var_name=local_cluster 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local instance_var_name=instance 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=1 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval local_cluster=cluster1 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ local_cluster=cluster1 2026-02-19T21:03:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ eval instance=1 2026-02-19T21:03:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ instance=1 2026-02-19T21:03:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-19T21:03:11.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:11.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:11.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-19T21:03:11.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-19T21:03:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-19T21:03:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_50 2026-02-19T21:03:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-19T21:03:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_50 2026-02-19T21:03:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local id_regexp= 2026-02-19T21:03:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-19T21:03:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-19T21:03:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:03:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-19T21:03:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 13 2026-02-19T21:03:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 13 2026-02-19T21:03:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:03:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=55983], [object_number=54, tag_tid=1, entry_tid=55982]]], state=connected] 2026-02-19T21:03:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=55983], [object_number=54, tag_tid=1, entry_tid=55982]]], state=connected] 2026-02-19T21:03:11.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-19T21:03:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-19T21:03:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_50 2026-02-19T21:03:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-19T21:03:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:03:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:03:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_50 2026-02-19T21:03:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local 'id_regexp=..*' 2026-02-19T21:03:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-19T21:03:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-19T21:03:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-19T21:03:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local status_log=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-19T21:03:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ tee /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:minimum_set: 13 2026-02-19T21:03:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:active_set: 13 2026-02-19T21:03:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:registered clients: 2026-02-19T21:03:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=55983], [object_number=54, tag_tid=1, entry_tid=55982]]], state=connected] 2026-02-19T21:03:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: [id=aebdaa49-1850-40e0-9a43-65c5f0c8a46c, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=55983], [object_number=54, tag_tid=1, entry_tid=55982]]], state=connected] 2026-02-19T21:03:11.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.status 2026-02-19T21:03:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ mirror_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-19T21:03:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-19T21:03:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2026-02-19T21:03:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:03:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:03:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:03:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:03:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-19T21:03:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 1 2026-02-19T21:03:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2026-02-19T21:03:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_50': 2026-02-19T21:03:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 128 MiB in 32 objects 2026-02-19T21:03:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:03:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 1 2026-02-19T21:03:13.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 11fc688d1e8a 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.11fc688d1e8a 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 20:51:28 2026 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 20:51:28 2026 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 20:52:29 2026 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: protected: False 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 11fc688d1e8a 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:03:13.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:03:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 4554daf0-7d99-480d-99ef-f1669f65d524 2026-02-19T21:03:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: false 2026-02-19T21:03:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ compare_image_snaps mirror image_50 snap 2026-02-19T21:03:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:03:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap_name=snap 2026-02-19T21:03:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local ret=0 2026-02-19T21:03:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local rmt_export=/tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.export 2026-02-19T21:03:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local loc_export=/tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_50.export 2026-02-19T21:03:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_50.export 2026-02-19T21:03:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.export 2026-02-19T21:03:13.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:03:13.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_50.export 2026-02-19T21:03:13.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-19T21:03:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cmp /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_50.export 2026-02-19T21:03:13.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -f /tmp/tmp.xpxdoJdHT7/cluster2-mirror-image_50.export /tmp/tmp.xpxdoJdHT7/cluster1-mirror-image_50.export 2026-02-19T21:03:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:13.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ testlog 'TEST: delete many images' 2026-02-19T21:03:13.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ tee -a /tmp/tmp.xpxdoJdHT7/rbd-mirror.test.log 2026-02-19T21:03:13.462 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ date '+%F %T' 2026-02-19T21:03:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo 2026-02-19 21:03:13 TEST: delete many images 2026-02-19T21:03:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:2026-02-19 21:03:13 TEST: delete many images 2026-02-19T21:03:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 50 2026-02-19T21:03:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_1 2026-02-19T21:03:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2026-02-19T21:03:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:03:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:13.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2026-02-19T21:03:13.895 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-19T21:03:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:03:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_1 2026-02-19T21:03:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:03:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-19T21:03:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-19T21:03:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_2 2026-02-19T21:03:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2026-02-19T21:03:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:03:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2026-02-19T21:03:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-19T21:03:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:03:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_2 2026-02-19T21:03:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:03:14.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-19T21:03:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-19T21:03:15.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_3 2026-02-19T21:03:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2026-02-19T21:03:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:03:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2026-02-19T21:03:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-19T21:03:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:03:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_3 2026-02-19T21:03:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:03:15.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-19T21:03:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-19T21:03:16.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_4 2026-02-19T21:03:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2026-02-19T21:03:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:03:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2026-02-19T21:03:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-19T21:03:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:03:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:16.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_4 2026-02-19T21:03:16.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:16.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:16.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:03:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-19T21:03:16.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-19T21:03:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_5 2026-02-19T21:03:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2026-02-19T21:03:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:03:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2026-02-19T21:03:17.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:17.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-19T21:03:17.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:17.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:17.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:03:17.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:17.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:17.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_5 2026-02-19T21:03:17.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:17.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:17.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:03:17.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-19T21:03:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-19T21:03:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_6 2026-02-19T21:03:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2026-02-19T21:03:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:03:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2026-02-19T21:03:18.911 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-19T21:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_6 2026-02-19T21:03:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:03:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-19T21:03:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-19T21:03:19.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_7 2026-02-19T21:03:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2026-02-19T21:03:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:03:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2026-02-19T21:03:19.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-19T21:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_7 2026-02-19T21:03:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:03:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-19T21:03:19.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-19T21:03:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_8 2026-02-19T21:03:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2026-02-19T21:03:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:03:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2026-02-19T21:03:20.877 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-19T21:03:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:03:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:20.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_8 2026-02-19T21:03:20.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:20.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:20.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:03:20.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-19T21:03:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-19T21:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_9 2026-02-19T21:03:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2026-02-19T21:03:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:03:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2026-02-19T21:03:21.876 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3594273850414672) > weight (0.25) 2026-02-19T21:03:21.876 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5705521796088214) > weight (0.25) 2026-02-19T21:03:21.876 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9471122507606409) > weight (0.25) 2026-02-19T21:03:21.877 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-19T21:03:21.877 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-19T21:03:21.877 INFO:tasks.rbd_mirror_thrash:waiting for 0.5558158945009395 secs before reviving daemons 2026-02-19T21:03:21.878 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:03:21.870+0000 7f3fc3939640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 87732) UID: 1000 2026-02-19T21:03:21.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-19T21:03:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:03:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_9 2026-02-19T21:03:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:03:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-19T21:03:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-19T21:03:22.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_10 2026-02-19T21:03:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2026-02-19T21:03:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:03:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2026-02-19T21:03:22.434 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-19T21:03:22.434 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T21:03:22.435 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-19T21:03:22.435 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-19T21:03:22.435 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-19T21:03:22.437 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-19T21:03:22.437 INFO:tasks.rbd_mirror_thrash:waiting for 102.11754002457873 secs before thrashing 2026-02-19T21:03:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-19T21:03:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:03:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_10 2026-02-19T21:03:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:03:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-19T21:03:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-19T21:03:23.067 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:03:23.058+0000 7ff7bdc78640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=709b76a5-68e5-40f2-a913-0f5552f2a4cc 2026-02-19T21:03:23.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:23.093 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_11 2026-02-19T21:03:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2026-02-19T21:03:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T21:03:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2026-02-19T21:03:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_11 2026-02-19T21:03:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T21:03:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_11 2026-02-19T21:03:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T21:03:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2026-02-19T21:03:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2026-02-19T21:03:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_12 2026-02-19T21:03:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2026-02-19T21:03:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T21:03:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2026-02-19T21:03:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_12 2026-02-19T21:03:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T21:03:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_12 2026-02-19T21:03:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T21:03:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2026-02-19T21:03:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2026-02-19T21:03:25.081 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:03:25.074+0000 7ff7bdc78640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=e3d71bea-907e-4390-b8ea-b164e5f8ebe0 2026-02-19T21:03:25.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_13 2026-02-19T21:03:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2026-02-19T21:03:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T21:03:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2026-02-19T21:03:25.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_13 2026-02-19T21:03:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T21:03:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_13 2026-02-19T21:03:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T21:03:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2026-02-19T21:03:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2026-02-19T21:03:26.026 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:03:26.018+0000 7ff7bd477640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=bb0fbbe0-269d-40de-85ab-495afa682240 2026-02-19T21:03:26.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_14 2026-02-19T21:03:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2026-02-19T21:03:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:26.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T21:03:26.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:26.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2026-02-19T21:03:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_14 2026-02-19T21:03:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T21:03:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_14 2026-02-19T21:03:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T21:03:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2026-02-19T21:03:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2026-02-19T21:03:27.063 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:03:27.057+0000 7fe4dd68c640 -1 reset not still connected to 0x564a23d4a0d0 2026-02-19T21:03:27.063 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:03:27.057+0000 7fe4dd68c640 -1 reset not still connected to 0x564a290e7860 2026-02-19T21:03:27.076 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:03:27.070+0000 7ff7bd477640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=d345ba20-24e6-4022-8fa4-2737b8391ecf 2026-02-19T21:03:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_15 2026-02-19T21:03:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2026-02-19T21:03:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T21:03:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2026-02-19T21:03:27.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_15 2026-02-19T21:03:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T21:03:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_15 2026-02-19T21:03:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T21:03:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2026-02-19T21:03:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2026-02-19T21:03:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_16 2026-02-19T21:03:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2026-02-19T21:03:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T21:03:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:28.296 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2026-02-19T21:03:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:28.947 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_16 2026-02-19T21:03:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T21:03:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_16 2026-02-19T21:03:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T21:03:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2026-02-19T21:03:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2026-02-19T21:03:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_17 2026-02-19T21:03:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2026-02-19T21:03:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T21:03:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2026-02-19T21:03:29.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:29.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_17 2026-02-19T21:03:29.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:29.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T21:03:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_17 2026-02-19T21:03:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T21:03:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2026-02-19T21:03:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2026-02-19T21:03:30.081 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:03:30.074+0000 7ff7bd477640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=eac262df-4bca-4f08-9721-bb777cecfa34 2026-02-19T21:03:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_18 2026-02-19T21:03:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2026-02-19T21:03:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T21:03:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2026-02-19T21:03:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_18 2026-02-19T21:03:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T21:03:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_18 2026-02-19T21:03:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T21:03:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2026-02-19T21:03:30.926 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2026-02-19T21:03:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_19 2026-02-19T21:03:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2026-02-19T21:03:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T21:03:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2026-02-19T21:03:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_19 2026-02-19T21:03:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T21:03:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_19 2026-02-19T21:03:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T21:03:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2026-02-19T21:03:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2026-02-19T21:03:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_20 2026-02-19T21:03:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2026-02-19T21:03:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T21:03:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2026-02-19T21:03:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:32.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_20 2026-02-19T21:03:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T21:03:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_20 2026-02-19T21:03:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T21:03:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2026-02-19T21:03:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2026-02-19T21:03:33.088 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:03:33.078+0000 7ff7bdc78640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=34aaf255-a028-4dda-90c0-ae0d48515507 2026-02-19T21:03:33.116 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_21 2026-02-19T21:03:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2026-02-19T21:03:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T21:03:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2026-02-19T21:03:33.905 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8321010638151974) > weight (0.25) 2026-02-19T21:03:33.905 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-19T21:03:33.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-19T21:03:33.905 INFO:tasks.rbd_mirror_thrash:waiting for 5.343792139594381 secs before reviving daemons 2026-02-19T21:03:33.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:03:33.898+0000 7f2302821640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 110128) UID: 1000 2026-02-19T21:03:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_21 2026-02-19T21:03:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T21:03:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_21 2026-02-19T21:03:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T21:03:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2026-02-19T21:03:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2026-02-19T21:03:34.104 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:03:34.098+0000 7ff7bdc78640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=2405118a-2b72-44cd-a98b-afad50e322fd 2026-02-19T21:03:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_22 2026-02-19T21:03:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2026-02-19T21:03:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T21:03:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2026-02-19T21:03:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:34.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_22 2026-02-19T21:03:34.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:34.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:34.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T21:03:34.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:34.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:34.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_22 2026-02-19T21:03:34.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:34.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:34.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T21:03:34.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2026-02-19T21:03:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2026-02-19T21:03:35.104 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:03:35.094+0000 7ff7bd477640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=dc9cbe3c-3c30-4147-97dc-5b4bf3a26964 2026-02-19T21:03:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_23 2026-02-19T21:03:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2026-02-19T21:03:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:35.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T21:03:35.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:35.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2026-02-19T21:03:35.880 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_23 2026-02-19T21:03:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:35.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:35.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T21:03:35.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:35.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:35.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_23 2026-02-19T21:03:35.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:35.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:35.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T21:03:35.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2026-02-19T21:03:35.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2026-02-19T21:03:36.064 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:03:36.054+0000 7ff7bdc78640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=aae73d36-e56f-4488-bc1c-eda2de4a4a64 2026-02-19T21:03:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_24 2026-02-19T21:03:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2026-02-19T21:03:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T21:03:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2026-02-19T21:03:36.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_24 2026-02-19T21:03:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T21:03:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_24 2026-02-19T21:03:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T21:03:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2026-02-19T21:03:37.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2026-02-19T21:03:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:37.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:37.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:37.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_25 2026-02-19T21:03:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2026-02-19T21:03:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T21:03:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2026-02-19T21:03:37.972 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_25 2026-02-19T21:03:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T21:03:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_25 2026-02-19T21:03:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T21:03:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2026-02-19T21:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2026-02-19T21:03:38.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_26 2026-02-19T21:03:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2026-02-19T21:03:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T21:03:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2026-02-19T21:03:38.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:38.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_26 2026-02-19T21:03:38.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:38.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:38.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T21:03:38.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:38.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:38.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_26 2026-02-19T21:03:38.974 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T21:03:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2026-02-19T21:03:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2026-02-19T21:03:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_27 2026-02-19T21:03:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2026-02-19T21:03:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T21:03:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2026-02-19T21:03:39.250 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-19T21:03:39.250 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T21:03:39.250 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-19T21:03:39.251 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-19T21:03:39.251 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-19T21:03:39.253 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-19T21:03:39.253 INFO:tasks.rbd_mirror_thrash:waiting for 80.11336277029709 secs before thrashing 2026-02-19T21:03:39.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_27 2026-02-19T21:03:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T21:03:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_27 2026-02-19T21:03:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T21:03:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2026-02-19T21:03:40.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2026-02-19T21:03:40.058 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:03:40.050+0000 7faaab268640 -1 rbd::mirror::ImageMap: 0x55fa5b2910e0 handle_peer_ack_remove: failed to remove global_image_id=bc93a362-0075-4b5e-8c78-6be5d453a095 2026-02-19T21:03:40.133 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_28 2026-02-19T21:03:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2026-02-19T21:03:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T21:03:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2026-02-19T21:03:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_28 2026-02-19T21:03:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T21:03:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_28 2026-02-19T21:03:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T21:03:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2026-02-19T21:03:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2026-02-19T21:03:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_29 2026-02-19T21:03:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2026-02-19T21:03:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T21:03:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2026-02-19T21:03:41.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_29 2026-02-19T21:03:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T21:03:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_29 2026-02-19T21:03:41.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:41.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:41.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T21:03:41.992 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2026-02-19T21:03:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2026-02-19T21:03:42.138 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_30 2026-02-19T21:03:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2026-02-19T21:03:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T21:03:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2026-02-19T21:03:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:42.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_30 2026-02-19T21:03:42.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:42.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:42.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T21:03:42.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:42.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_30 2026-02-19T21:03:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T21:03:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2026-02-19T21:03:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2026-02-19T21:03:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_31 2026-02-19T21:03:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2026-02-19T21:03:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T21:03:43.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:43.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2026-02-19T21:03:43.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_31 2026-02-19T21:03:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T21:03:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:43.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_31 2026-02-19T21:03:43.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:43.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:43.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T21:03:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2026-02-19T21:03:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2026-02-19T21:03:44.079 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:03:44.070+0000 7faaab268640 -1 rbd::mirror::ImageMap: 0x55fa5b2910e0 handle_peer_ack_remove: failed to remove global_image_id=96b77a01-802d-450a-965e-b51abc0bfe7b 2026-02-19T21:03:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_32 2026-02-19T21:03:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2026-02-19T21:03:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T21:03:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2026-02-19T21:03:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_32 2026-02-19T21:03:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T21:03:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:45.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:45.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_32 2026-02-19T21:03:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T21:03:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2026-02-19T21:03:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2026-02-19T21:03:45.078 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:03:45.070+0000 7faaab268640 -1 rbd::mirror::ImageMap: 0x55fa5b2910e0 handle_peer_ack_remove: failed to remove global_image_id=3a18f5f4-b23c-4fbc-bbec-0dff64ec1f7b 2026-02-19T21:03:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:45.155 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_33 2026-02-19T21:03:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2026-02-19T21:03:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T21:03:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2026-02-19T21:03:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_33 2026-02-19T21:03:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T21:03:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:45.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_33 2026-02-19T21:03:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T21:03:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2026-02-19T21:03:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2026-02-19T21:03:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_34 2026-02-19T21:03:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2026-02-19T21:03:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T21:03:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2026-02-19T21:03:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_34 2026-02-19T21:03:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T21:03:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_34 2026-02-19T21:03:47.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:47.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:47.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T21:03:47.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2026-02-19T21:03:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2026-02-19T21:03:47.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_35 2026-02-19T21:03:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2026-02-19T21:03:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T21:03:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2026-02-19T21:03:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:48.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_35 2026-02-19T21:03:48.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:48.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:48.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T21:03:48.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:48.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_35 2026-02-19T21:03:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T21:03:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2026-02-19T21:03:48.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2026-02-19T21:03:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_36 2026-02-19T21:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2026-02-19T21:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T21:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2026-02-19T21:03:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_36 2026-02-19T21:03:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T21:03:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_36 2026-02-19T21:03:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T21:03:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2026-02-19T21:03:49.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2026-02-19T21:03:49.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:03:49.078+0000 7faaab268640 -1 rbd::mirror::ImageMap: 0x55fa5b2910e0 handle_peer_ack_remove: failed to remove global_image_id=85d99ed8-1ae5-4a5e-94ca-27418c991bb9 2026-02-19T21:03:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_37 2026-02-19T21:03:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2026-02-19T21:03:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:03:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2026-02-19T21:03:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_37 2026-02-19T21:03:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:03:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:50.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:50.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_37 2026-02-19T21:03:50.020 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:03:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2026-02-19T21:03:50.056 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2026-02-19T21:03:50.095 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:03:50.086+0000 7faaab268640 -1 rbd::mirror::ImageMap: 0x55fa5b2910e0 handle_peer_ack_remove: failed to remove global_image_id=476b8ccd-9dbe-4762-b2da-c13d92b3bc09 2026-02-19T21:03:50.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_38 2026-02-19T21:03:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2026-02-19T21:03:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:03:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2026-02-19T21:03:50.878 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_38 2026-02-19T21:03:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:03:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_38 2026-02-19T21:03:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:03:50.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2026-02-19T21:03:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2026-02-19T21:03:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_39 2026-02-19T21:03:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2026-02-19T21:03:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:03:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2026-02-19T21:03:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:52.014 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_39 2026-02-19T21:03:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:03:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_39 2026-02-19T21:03:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:03:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2026-02-19T21:03:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2026-02-19T21:03:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_40 2026-02-19T21:03:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2026-02-19T21:03:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:03:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2026-02-19T21:03:53.008 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:53.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_40 2026-02-19T21:03:53.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:53.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:53.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:03:53.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:53.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:53.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_40 2026-02-19T21:03:53.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:53.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:53.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:03:53.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2026-02-19T21:03:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2026-02-19T21:03:53.174 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_41 2026-02-19T21:03:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2026-02-19T21:03:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:03:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2026-02-19T21:03:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:54.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_41 2026-02-19T21:03:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:03:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_41 2026-02-19T21:03:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:03:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2026-02-19T21:03:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2026-02-19T21:03:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_42 2026-02-19T21:03:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2026-02-19T21:03:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:03:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:54.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2026-02-19T21:03:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_42 2026-02-19T21:03:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:03:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_42 2026-02-19T21:03:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:03:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2026-02-19T21:03:55.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2026-02-19T21:03:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_43 2026-02-19T21:03:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2026-02-19T21:03:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:03:55.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:55.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2026-02-19T21:03:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_43 2026-02-19T21:03:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:03:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:55.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_43 2026-02-19T21:03:55.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:55.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:55.902 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:03:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2026-02-19T21:03:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2026-02-19T21:03:56.018 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:03:56.009+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T21:03:56.019 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:03:56.013+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T21:03:56.019 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:03:56.013+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T21:03:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_44 2026-02-19T21:03:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2026-02-19T21:03:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:03:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2026-02-19T21:03:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:57.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_44 2026-02-19T21:03:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:03:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_44 2026-02-19T21:03:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:03:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2026-02-19T21:03:57.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2026-02-19T21:03:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_45 2026-02-19T21:03:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2026-02-19T21:03:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:03:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2026-02-19T21:03:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_45 2026-02-19T21:03:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:03:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_45 2026-02-19T21:03:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:03:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2026-02-19T21:03:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2026-02-19T21:03:58.317 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:03:58.310+0000 7faaab268640 -1 rbd::mirror::ImageMap: 0x55fa5b2910e0 handle_peer_ack_remove: failed to remove global_image_id=ea02bcb2-4a9c-449e-beda-ecd26614ee2d 2026-02-19T21:03:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_46 2026-02-19T21:03:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2026-02-19T21:03:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:58.404 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:03:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2026-02-19T21:03:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:03:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_46 2026-02-19T21:03:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:03:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:03:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:03:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_46 2026-02-19T21:03:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:03:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2026-02-19T21:03:59.071 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2026-02-19T21:03:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:03:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:03:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:03:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_47 2026-02-19T21:03:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2026-02-19T21:03:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:03:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:03:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:03:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:03:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2026-02-19T21:04:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:04:00.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_47 2026-02-19T21:04:00.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:00.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:00.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:04:00.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:04:00.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:04:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_47 2026-02-19T21:04:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:04:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2026-02-19T21:04:00.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2026-02-19T21:04:00.200 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:04:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_48 2026-02-19T21:04:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2026-02-19T21:04:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:04:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:04:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2026-02-19T21:04:00.882 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:04:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_48 2026-02-19T21:04:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:04:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:04:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:04:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_48 2026-02-19T21:04:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:04:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2026-02-19T21:04:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2026-02-19T21:04:01.054 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:04:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_49 2026-02-19T21:04:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2026-02-19T21:04:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:04:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:04:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2026-02-19T21:04:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:04:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_49 2026-02-19T21:04:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:04:02.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:04:02.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:04:02.050 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_49 2026-02-19T21:04:02.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:02.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:02.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:04:02.051 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2026-02-19T21:04:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2026-02-19T21:04:02.129 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:04:02.122+0000 7faaaba69640 -1 rbd::mirror::ImageMap: 0x55fa5b2910e0 handle_peer_ack_remove: failed to remove global_image_id=0a39469c-5889-44f4-9724-5da19a7298c8 2026-02-19T21:04:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:04:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_50 2026-02-19T21:04:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2026-02-19T21:04:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:04:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local snap=snap 2026-02-19T21:04:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2026-02-19T21:04:03.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing snap: 100% complete...done. 2026-02-19T21:04:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_50 2026-02-19T21:04:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:04:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:04:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:04:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_50 2026-02-19T21:04:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:04:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2026-02-19T21:04:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2026-02-19T21:04:03.118 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:04:03.110+0000 7faaaba69640 -1 rbd::mirror::ImageMap: 0x55fa5b2910e0 handle_peer_ack_remove: failed to remove global_image_id=4554daf0-7d99-480d-99ef-f1669f65d524 2026-02-19T21:04:03.196 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-19T21:04:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ testlog 'TEST: image deletions should propagate' 2026-02-19T21:04:03.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ tee -a /tmp/tmp.xpxdoJdHT7/rbd-mirror.test.log 2026-02-19T21:04:03.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ date '+%F %T' 2026-02-19T21:04:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo 2026-02-19 21:04:03 TEST: image deletions should propagate 2026-02-19T21:04:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:2026-02-19 21:04:03 TEST: image deletions should propagate 2026-02-19T21:04:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_pool_images cluster1 mirror 0 2026-02-19T21:04:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_count=0 2026-02-19T21:04:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local count 2026-02-19T21:04:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local last_count=0 2026-02-19T21:04:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:04:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 40 2026-02-19T21:04:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in `seq 1 40` 2026-02-19T21:04:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 1 -ne 1 2026-02-19T21:04:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-19T21:04:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep 'images: ' 2026-02-19T21:04:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut '-d ' -f 2 2026-02-19T21:04:03.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: failed to query services: (13) Permission denied 2026-02-19T21:04:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ count=8 2026-02-19T21:04:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 8 = 0 2026-02-19T21:04:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 8 -ne 0 2026-02-19T21:04:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ break 2026-02-19T21:04:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 8 -eq 0 2026-02-19T21:04:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ last_count=8 2026-02-19T21:04:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ true 2026-02-19T21:04:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 40 2026-02-19T21:04:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in `seq 1 40` 2026-02-19T21:04:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 1 -ne 1 2026-02-19T21:04:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-19T21:04:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep 'images: ' 2026-02-19T21:04:03.236 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut '-d ' -f 2 2026-02-19T21:04:03.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: failed to query services: (13) Permission denied 2026-02-19T21:04:03.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ count=8 2026-02-19T21:04:03.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 8 = 0 2026-02-19T21:04:03.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 8 -ne 8 2026-02-19T21:04:03.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in `seq 1 40` 2026-02-19T21:04:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 2 -ne 1 2026-02-19T21:04:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 30 2026-02-19T21:04:33.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-19T21:04:33.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep 'images: ' 2026-02-19T21:04:33.263 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut '-d ' -f 2 2026-02-19T21:04:33.287 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: failed to query services: (13) Permission denied 2026-02-19T21:04:33.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ count=0 2026-02-19T21:04:33.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 = 0 2026-02-19T21:04:33.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:33.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2026-02-19T21:04:33.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state 2026-02-19T21:04:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 40 2026-02-19T21:04:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in `seq 1 40` 2026-02-19T21:04:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 1 -ne 1 2026-02-19T21:04:33.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-19T21:04:33.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep 'image health:' 2026-02-19T21:04:33.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cut '-d ' -f 3 2026-02-19T21:04:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: failed to query services: (13) Permission denied 2026-02-19T21:04:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ state=OK 2026-02-19T21:04:33.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test OK = ERROR 2026-02-19T21:04:33.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test OK = OK 2026-02-19T21:04:33.320 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 50 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_1 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_1 2026-02-19T21:04:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T21:04:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-19T21:04:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-19T21:04:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-19T21:04:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:04:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_1 2026-02-19T21:04:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T21:04:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-19T21:04:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:33.468 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-19T21:04:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_2 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_2 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T21:04:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-19T21:04:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:33.495 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-19T21:04:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-19T21:04:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:04:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_2 2026-02-19T21:04:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:33.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:33.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T21:04:33.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-19T21:04:33.601 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:33.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-19T21:04:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_3 2026-02-19T21:04:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-19T21:04:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_3 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-19T21:04:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-19T21:04:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_3 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T21:04:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-19T21:04:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-19T21:04:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_4 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_4 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:33.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_4 2026-02-19T21:04:33.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-19T21:04:33.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-19T21:04:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-19T21:04:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:04:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_4 2026-02-19T21:04:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_4 2026-02-19T21:04:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-19T21:04:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_5 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:04:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_5 2026-02-19T21:04:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T21:04:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-19T21:04:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-19T21:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-19T21:04:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_5 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-19T21:04:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-19T21:04:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_6 2026-02-19T21:04:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_6 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T21:04:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-19T21:04:34.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-19T21:04:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_6 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T21:04:34.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-19T21:04:34.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:34.235 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-19T21:04:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_7 2026-02-19T21:04:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-19T21:04:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_7 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-19T21:04:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-19T21:04:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-19T21:04:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:04:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_7 2026-02-19T21:04:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T21:04:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-19T21:04:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:34.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-19T21:04:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_8 2026-02-19T21:04:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-19T21:04:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_8 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-19T21:04:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-19T21:04:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:34.517 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-19T21:04:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:04:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_8 2026-02-19T21:04:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:34.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T21:04:34.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-19T21:04:34.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-19T21:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_9 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_9 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T21:04:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-19T21:04:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-19T21:04:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-19T21:04:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:04:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:34.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:34.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_9 2026-02-19T21:04:34.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:34.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:34.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T21:04:34.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-19T21:04:34.670 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:34.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-19T21:04:34.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:34.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_10 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_10 2026-02-19T21:04:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T21:04:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-19T21:04:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-19T21:04:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:34.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:34.923 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:35.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-19T21:04:35.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:04:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_10 2026-02-19T21:04:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T21:04:35.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-19T21:04:35.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-19T21:04:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_11 2026-02-19T21:04:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2026-02-19T21:04:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T21:04:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_11 2026-02-19T21:04:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T21:04:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-19T21:04:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-19T21:04:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_11 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_11 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_11 2026-02-19T21:04:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-19T21:04:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.199 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-19T21:04:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_12 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_12 2026-02-19T21:04:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T21:04:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-19T21:04:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-19T21:04:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2026-02-19T21:04:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_12 2026-02-19T21:04:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_12 2026-02-19T21:04:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_12 2026-02-19T21:04:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-19T21:04:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-19T21:04:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_13 2026-02-19T21:04:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2026-02-19T21:04:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_13 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-19T21:04:35.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.375 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-19T21:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2026-02-19T21:04:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_13 2026-02-19T21:04:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_13 2026-02-19T21:04:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_13 2026-02-19T21:04:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-19T21:04:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-19T21:04:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_14 2026-02-19T21:04:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2026-02-19T21:04:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T21:04:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_14 2026-02-19T21:04:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T21:04:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-19T21:04:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-19T21:04:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:35.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2026-02-19T21:04:35.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_14 2026-02-19T21:04:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_14 2026-02-19T21:04:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_14 2026-02-19T21:04:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-19T21:04:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.652 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-19T21:04:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_15 2026-02-19T21:04:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_15 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T21:04:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-19T21:04:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-19T21:04:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:35.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2026-02-19T21:04:35.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_15 2026-02-19T21:04:35.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:35.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:35.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:35.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_15 2026-02-19T21:04:35.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_15 2026-02-19T21:04:35.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-19T21:04:35.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-19T21:04:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_16 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_16 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_16 2026-02-19T21:04:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-19T21:04:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-19T21:04:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2026-02-19T21:04:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_16 2026-02-19T21:04:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_16 2026-02-19T21:04:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_16 2026-02-19T21:04:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-19T21:04:35.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-19T21:04:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_17 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_17 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T21:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-19T21:04:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-19T21:04:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2026-02-19T21:04:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_17 2026-02-19T21:04:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_17 2026-02-19T21:04:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_17 2026-02-19T21:04:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-19T21:04:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-19T21:04:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_18 2026-02-19T21:04:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2026-02-19T21:04:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_18 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-19T21:04:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-19T21:04:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2026-02-19T21:04:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_18 2026-02-19T21:04:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_18 2026-02-19T21:04:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_18 2026-02-19T21:04:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-19T21:04:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:36.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-19T21:04:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_19 2026-02-19T21:04:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2026-02-19T21:04:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:36.470 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T21:04:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_19 2026-02-19T21:04:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T21:04:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-19T21:04:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-19T21:04:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2026-02-19T21:04:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_19 2026-02-19T21:04:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:36.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:36.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:36.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_19 2026-02-19T21:04:36.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:36.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:36.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_19 2026-02-19T21:04:36.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-19T21:04:36.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:36.616 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-19T21:04:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_20 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_20 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T21:04:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-19T21:04:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:36.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-19T21:04:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2026-02-19T21:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_20 2026-02-19T21:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_20 2026-02-19T21:04:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_20 2026-02-19T21:04:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-19T21:04:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-19T21:04:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_21 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_21 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T21:04:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-19T21:04:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-19T21:04:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2026-02-19T21:04:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_21 2026-02-19T21:04:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_21 2026-02-19T21:04:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_21 2026-02-19T21:04:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-19T21:04:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:37.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-19T21:04:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_22 2026-02-19T21:04:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2026-02-19T21:04:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_22 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-19T21:04:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:37.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-19T21:04:37.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:37.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:37.357 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2026-02-19T21:04:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_22 2026-02-19T21:04:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_22 2026-02-19T21:04:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_22 2026-02-19T21:04:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-19T21:04:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:37.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-19T21:04:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_23 2026-02-19T21:04:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2026-02-19T21:04:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:37.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T21:04:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_23 2026-02-19T21:04:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T21:04:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-19T21:04:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-19T21:04:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2026-02-19T21:04:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_23 2026-02-19T21:04:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_23 2026-02-19T21:04:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:37.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_23 2026-02-19T21:04:37.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-19T21:04:37.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-19T21:04:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_24 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_24 2026-02-19T21:04:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T21:04:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-19T21:04:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:37.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-19T21:04:37.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:37.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:37.660 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_24 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_24 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_24 2026-02-19T21:04:37.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-19T21:04:37.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_25 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_25 2026-02-19T21:04:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T21:04:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-19T21:04:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:37.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-19T21:04:37.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:37.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:37.810 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2026-02-19T21:04:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_25 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_25 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_25 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-19T21:04:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-19T21:04:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_26 2026-02-19T21:04:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_26 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T21:04:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-19T21:04:37.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-19T21:04:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_26 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_26 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_26 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-19T21:04:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_27 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_27 2026-02-19T21:04:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T21:04:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-19T21:04:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.111 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-19T21:04:38.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:38.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:38.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2026-02-19T21:04:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_27 2026-02-19T21:04:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_27 2026-02-19T21:04:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_27 2026-02-19T21:04:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-19T21:04:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-19T21:04:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_28 2026-02-19T21:04:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2026-02-19T21:04:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T21:04:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_28 2026-02-19T21:04:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_28 2026-02-19T21:04:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-19T21:04:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-19T21:04:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2026-02-19T21:04:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_28 2026-02-19T21:04:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_28 2026-02-19T21:04:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_28 2026-02-19T21:04:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-19T21:04:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.391 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-19T21:04:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_29 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_29 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T21:04:38.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-19T21:04:38.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-19T21:04:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2026-02-19T21:04:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_29 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_29 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_29 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-19T21:04:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-19T21:04:38.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:38.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.548 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_30 2026-02-19T21:04:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2026-02-19T21:04:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T21:04:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_30 2026-02-19T21:04:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T21:04:38.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-19T21:04:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-19T21:04:38.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:38.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:38.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2026-02-19T21:04:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_30 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_30 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_30 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-19T21:04:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-19T21:04:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_31 2026-02-19T21:04:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_31 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T21:04:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-19T21:04:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-19T21:04:38.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:38.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:38.724 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2026-02-19T21:04:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_31 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_31 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_31 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-19T21:04:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.847 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-19T21:04:38.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:38.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.851 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_32 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_32 2026-02-19T21:04:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T21:04:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-19T21:04:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.871 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-19T21:04:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2026-02-19T21:04:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_32 2026-02-19T21:04:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_32 2026-02-19T21:04:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:38.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_32 2026-02-19T21:04:38.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-19T21:04:38.978 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:38.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-19T21:04:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_33 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_33 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T21:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-19T21:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-19T21:04:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2026-02-19T21:04:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_33 2026-02-19T21:04:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_33 2026-02-19T21:04:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_33 2026-02-19T21:04:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-19T21:04:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-19T21:04:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_34 2026-02-19T21:04:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2026-02-19T21:04:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T21:04:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_34 2026-02-19T21:04:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T21:04:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-19T21:04:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-19T21:04:39.176 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:39.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:39.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2026-02-19T21:04:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_34 2026-02-19T21:04:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_34 2026-02-19T21:04:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_34 2026-02-19T21:04:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-19T21:04:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-19T21:04:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_35 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_35 2026-02-19T21:04:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_35 2026-02-19T21:04:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-19T21:04:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-19T21:04:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:39.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:39.327 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2026-02-19T21:04:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_35 2026-02-19T21:04:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_35 2026-02-19T21:04:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_35 2026-02-19T21:04:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-19T21:04:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-19T21:04:39.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_36 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_36 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_36 2026-02-19T21:04:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-19T21:04:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.477 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-19T21:04:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:39.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_36 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_36 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_36 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-19T21:04:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.603 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-19T21:04:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_37 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_37 2026-02-19T21:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-19T21:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-19T21:04:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2026-02-19T21:04:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_37 2026-02-19T21:04:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_37 2026-02-19T21:04:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_37 2026-02-19T21:04:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-19T21:04:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.757 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-19T21:04:39.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_38 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:39.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_38 2026-02-19T21:04:39.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_38 2026-02-19T21:04:39.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-19T21:04:39.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-19T21:04:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2026-02-19T21:04:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_38 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_38 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_38 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-19T21:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.910 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-19T21:04:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_39 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_39 2026-02-19T21:04:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:04:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-19T21:04:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:39.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-19T21:04:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2026-02-19T21:04:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_39 2026-02-19T21:04:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_39 2026-02-19T21:04:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.042 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_39 2026-02-19T21:04:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-19T21:04:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-19T21:04:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_40 2026-02-19T21:04:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2026-02-19T21:04:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:04:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_40 2026-02-19T21:04:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:04:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-19T21:04:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-19T21:04:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2026-02-19T21:04:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_40 2026-02-19T21:04:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_40 2026-02-19T21:04:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_40 2026-02-19T21:04:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-19T21:04:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.216 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-19T21:04:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_41 2026-02-19T21:04:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2026-02-19T21:04:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:04:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_41 2026-02-19T21:04:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:04:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-19T21:04:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-19T21:04:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:40.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_41 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_41 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_41 2026-02-19T21:04:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-19T21:04:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.368 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-19T21:04:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_42 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_42 2026-02-19T21:04:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:04:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-19T21:04:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-19T21:04:40.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:40.396 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:40.397 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2026-02-19T21:04:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_42 2026-02-19T21:04:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_42 2026-02-19T21:04:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_42 2026-02-19T21:04:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-19T21:04:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-19T21:04:40.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:40.524 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:40.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:40.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:40.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_43 2026-02-19T21:04:40.525 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2026-02-19T21:04:40.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:04:40.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:40.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:40.526 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_43 2026-02-19T21:04:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:04:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-19T21:04:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-19T21:04:40.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:40.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:40.547 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:40.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2026-02-19T21:04:40.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.649 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_43 2026-02-19T21:04:40.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:40.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:40.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:40.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_43 2026-02-19T21:04:40.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.650 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_43 2026-02-19T21:04:40.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-19T21:04:40.651 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.673 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-19T21:04:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_44 2026-02-19T21:04:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2026-02-19T21:04:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:04:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_44 2026-02-19T21:04:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:04:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-19T21:04:40.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.698 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-19T21:04:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:40.701 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:40.702 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2026-02-19T21:04:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_44 2026-02-19T21:04:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_44 2026-02-19T21:04:40.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_44 2026-02-19T21:04:40.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-19T21:04:40.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-19T21:04:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_45 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_45 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:04:40.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-19T21:04:40.830 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.850 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-19T21:04:40.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:40.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:40.853 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2026-02-19T21:04:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_45 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_45 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_45 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-19T21:04:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-19T21:04:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_46 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_46 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:04:40.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-19T21:04:40.983 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-19T21:04:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_46 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_46 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_46 2026-02-19T21:04:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-19T21:04:41.110 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-19T21:04:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_47 2026-02-19T21:04:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2026-02-19T21:04:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_47 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-19T21:04:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:41.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-19T21:04:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2026-02-19T21:04:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_47 2026-02-19T21:04:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_47 2026-02-19T21:04:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_47 2026-02-19T21:04:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-19T21:04:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:41.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-19T21:04:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_48 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_48 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_48 2026-02-19T21:04:41.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-19T21:04:41.285 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:41.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-19T21:04:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:41.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2026-02-19T21:04:41.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_48 2026-02-19T21:04:41.410 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_48 2026-02-19T21:04:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_48 2026-02-19T21:04:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-19T21:04:41.412 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:41.431 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-19T21:04:41.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:41.435 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_49 2026-02-19T21:04:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2026-02-19T21:04:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_49 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-19T21:04:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-19T21:04:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:41.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2026-02-19T21:04:41.560 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_49 2026-02-19T21:04:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_49 2026-02-19T21:04:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_49 2026-02-19T21:04:41.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-19T21:04:41.562 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-19T21:04:41.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_50 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:04:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:41.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:41.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:41.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_50 2026-02-19T21:04:41.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:41.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:41.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_50 2026-02-19T21:04:41.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-19T21:04:41.590 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-19T21:04:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:04:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2026-02-19T21:04:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_50 2026-02-19T21:04:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:04:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_50 2026-02-19T21:04:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_50 2026-02-19T21:04:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-19T21:04:41.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-19T21:04:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:04:41.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:41.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:04:41.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:41.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ testlog 'TEST: delete images during bootstrap' 2026-02-19T21:04:41.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ tee -a /tmp/tmp.xpxdoJdHT7/rbd-mirror.test.log 2026-02-19T21:04:41.740 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ date '+%F %T' 2026-02-19T21:04:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo 2026-02-19 21:04:41 TEST: delete images during bootstrap 2026-02-19T21:04:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:2026-02-19 21:04:41 TEST: delete images during bootstrap 2026-02-19T21:04:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_pool_mirror_mode cluster1 mirror image 2026-02-19T21:04:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=image 2026-02-19T21:04:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2026-02-19T21:04:41.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:note: changing mirroring mode from pool to image 2026-02-19T21:04:41.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-19T21:04:41.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:41.770 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=image 2026-02-19T21:04:41.771 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-19T21:04:41.797 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:note: changing mirroring mode from pool to image 2026-02-19T21:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ start_mirror cluster1 2026-02-19T21:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:41.805 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set_cluster_instance cluster1 cluster instance 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local val=cluster1 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster_var_name=cluster 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local instance_var_name=instance 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cluster=cluster1 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=cluster1 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' cluster1 = cluster1 ']' 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ instance=0 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval cluster=cluster1 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ cluster=cluster1 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ eval instance=0 2026-02-19T21:04:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ instance=0 2026-02-19T21:04:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T21:04:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T21:04:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=test 2026-02-19T21:04:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 1 10 2026-02-19T21:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T21:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_1 2026-02-19T21:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_1 512M 2026-02-19T21:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:04:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:04:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:04:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:04:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T21:04:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T21:04:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T21:04:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:04:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_1 2026-02-19T21:04:41.844 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ enable_mirror cluster2 mirror image_1 2026-02-19T21:04:41.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:41.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:04:41.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T21:04:41.845 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 journal 2026-02-19T21:04:41.879 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring enabled 2026-02-19T21:04:41.885 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2026-02-19T21:04:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_1': 2026-02-19T21:04:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T21:04:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:04:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 0 2026-02-19T21:04:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 34708758e851 2026-02-19T21:04:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.34708758e851 2026-02-19T21:04:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:04:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:04:41.916 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:04:41.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:04:41.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 21:04:41 2026 2026-02-19T21:04:41.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 21:04:41 2026 2026-02-19T21:04:41.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:04:41 2026 2026-02-19T21:04:41.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 34708758e851 2026-02-19T21:04:41.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:04:41.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:04:41.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: be1f02a5-ad12-416b-83f9-0621b4962a29 2026-02-19T21:04:41.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: true 2026-02-19T21:04:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-19T21:04:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:04:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T21:04:41.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=11 2026-02-19T21:04:41.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:04:41.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-19T21:04:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 744448 bytes 2026-02-19T21:04:42.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 13339648 bytes 2026-02-19T21:04:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 40003584 bytes 2026-02-19T21:04:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 47026176 bytes 2026-02-19T21:04:43.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 101592064 bytes 2026-02-19T21:04:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 145855488 bytes 2026-02-19T21:04:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 218746880 bytes 2026-02-19T21:04:46.134 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 286453760 bytes 2026-02-19T21:04:46.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 309307392 bytes 2026-02-19T21:04:47.299 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 383925248 bytes 2026-02-19T21:04:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 394394624 bytes 2026-02-19T21:04:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 427582464 bytes 2026-02-19T21:04:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 506711040 bytes 2026-02-19T21:04:50.877 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T21:04:50.870+0000 7f5ce3652640 -1 reset not still connected to 0x56104f0dbd40 2026-02-19T21:04:50.878 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T21:04:50.870+0000 7f5ce3652640 -1 reset not still connected to 0x5610501924e0 2026-02-19T21:04:50.878 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T21:04:50.870+0000 7f5ce3652640 -1 reset not still connected to 0x561051bcd040 2026-02-19T21:04:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 542593024 bytes 2026-02-19T21:04:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 575822848 bytes 2026-02-19T21:04:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T21:04:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:04:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T21:04:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2026-02-19T21:04:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:04:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=present 2026-02-19T21:04:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:04:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:04:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:04:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_1 2026-02-19T21:04:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T21:04:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-19T21:04:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=11fcbd17a4dc 2026-02-19T21:04:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:04:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:04:53.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_1 present 11fcbd17a4dc 2026-02-19T21:04:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:04:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:04:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=present 2026-02-19T21:04:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=11fcbd17a4dc 2026-02-19T21:04:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:04:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:04:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_1 2026-02-19T21:04:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:04:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:04:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T21:04:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-19T21:04:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:04:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=11fcbd17a4dc 2026-02-19T21:04:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 11fcbd17a4dc 2026-02-19T21:04:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 11fcbd17a4dc -o 11fcbd17a4dc = 11fcbd17a4dc 2026-02-19T21:04:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T21:04:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test present = present 2026-02-19T21:04:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:04:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ disable_mirror cluster2 mirror image_1 2026-02-19T21:04:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:04:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:04:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:04:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2026-02-19T21:04:59.368 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5039713886837762) > weight (0.25) 2026-02-19T21:04:59.368 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7852771818524038) > weight (0.25) 2026-02-19T21:04:59.368 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.4460589180271247) > weight (0.25) 2026-02-19T21:04:59.368 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.41745500149146797) > weight (0.25) 2026-02-19T21:04:59.369 INFO:tasks.rbd_mirror_thrash:waiting for 103.98578906363116 secs before thrashing 2026-02-19T21:05:04.556 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3886005700019245) > weight (0.25) 2026-02-19T21:05:04.556 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3329268548510902) > weight (0.25) 2026-02-19T21:05:04.556 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5235739171607816) > weight (0.25) 2026-02-19T21:05:04.557 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.36631614378685795) > weight (0.25) 2026-02-19T21:05:04.557 INFO:tasks.rbd_mirror_thrash:waiting for 60.603005180269506 secs before thrashing 2026-02-19T21:05:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring disabled 2026-02-19T21:05:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-19T21:05:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:05:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:05:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:05:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:05:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:05:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:05:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_1 2026-02-19T21:05:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:05:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:05:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T21:05:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-19T21:05:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:05:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-19T21:05:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:05:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:05:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:05:23.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-19T21:05:23.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:05:23.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:23.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:05:23.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:05:23.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:05:23.127 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:05:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:05:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_1 2026-02-19T21:05:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:05:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:05:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_1 2026-02-19T21:05:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-19T21:05:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:05:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-19T21:05:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:05:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:05:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:05:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:05:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ purge_snapshots cluster2 mirror image_1 2026-02-19T21:05:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:05:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:05:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2026-02-19T21:05:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-19T21:05:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:05:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:05:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:05:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:05:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_1 2026-02-19T21:05:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:05:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:23.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_1 2026-02-19T21:05:23.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-19T21:05:23.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-19T21:05:23.348 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T21:05:23.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:05:23.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T21:05:23.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_2 2026-02-19T21:05:23.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_2 512M 2026-02-19T21:05:23.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:05:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:05:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:05:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:05:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:05:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T21:05:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T21:05:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T21:05:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:05:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_2 2026-02-19T21:05:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ enable_mirror cluster2 mirror image_2 2026-02-19T21:05:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:05:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:05:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T21:05:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 journal 2026-02-19T21:05:23.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring enabled 2026-02-19T21:05:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2026-02-19T21:05:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_2': 2026-02-19T21:05:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 0 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 3488a4c1b941 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.3488a4c1b941 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 21:05:23 2026 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 21:05:23 2026 2026-02-19T21:05:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:05:23 2026 2026-02-19T21:05:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 3488a4c1b941 2026-02-19T21:05:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:05:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:05:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 75a056be-d927-4ac2-9aa3-08390590245c 2026-02-19T21:05:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: true 2026-02-19T21:05:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-19T21:05:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:05:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:23.455 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:05:23.456 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T21:05:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=14 2026-02-19T21:05:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:05:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-19T21:05:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 30662656 bytes 2026-02-19T21:05:26.440 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 96885760 bytes 2026-02-19T21:05:26.881 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 117262336 bytes 2026-02-19T21:05:26.967 INFO:tasks.ceph.cluster2.osd.1.trial151.stderr:2026-02-19T21:05:26.962+0000 7f58ec161640 -1 reset not still connected to 0x5561a35e4a90 2026-02-19T21:05:27.095 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 125753344 bytes 2026-02-19T21:05:28.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 198381568 bytes 2026-02-19T21:05:28.812 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 239815680 bytes 2026-02-19T21:05:29.559 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 286227456 bytes 2026-02-19T21:05:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 361299968 bytes 2026-02-19T21:05:30.932 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 396274688 bytes 2026-02-19T21:05:30.994 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 417759232 bytes 2026-02-19T21:05:31.734 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 457849856 bytes 2026-02-19T21:05:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 497639424 bytes 2026-02-19T21:05:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 509995008 bytes 2026-02-19T21:05:33.387 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 552975360 bytes 2026-02-19T21:05:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 590371840 bytes 2026-02-19T21:05:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 604848128 bytes 2026-02-19T21:05:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 627269632 bytes 2026-02-19T21:05:35.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 652541952 bytes 2026-02-19T21:05:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 723407872 bytes 2026-02-19T21:05:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2000 IOs, 762677248 bytes 2026-02-19T21:05:37.117 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2100 IOs, 773328896 bytes 2026-02-19T21:05:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T21:05:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:05:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T21:05:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:05:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2026-02-19T21:05:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=present 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_2 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-19T21:05:37.467 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:05:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=11fc22ff70e1 2026-02-19T21:05:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:05:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:05:37.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_2 present 11fc22ff70e1 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=present 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=11fc22ff70e1 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_2 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-19T21:05:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=11fc22ff70e1 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 11fc22ff70e1 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 11fc22ff70e1 -o 11fc22ff70e1 = 11fc22ff70e1 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test present = present 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ disable_mirror cluster2 mirror image_2 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:05:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2026-02-19T21:06:05.160 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9650930059759906) > weight (0.25) 2026-02-19T21:06:05.160 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-19T21:06:05.161 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-19T21:06:05.161 INFO:tasks.rbd_mirror_thrash:waiting for 9.854396103539552 secs before reviving daemons 2026-02-19T21:06:05.162 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:06:05.154+0000 7fdb2e693640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 97725) UID: 1000 2026-02-19T21:06:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring disabled 2026-02-19T21:06:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-19T21:06:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:06:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:06:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:06:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:06:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:06:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:06:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_2 2026-02-19T21:06:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:06:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:06:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T21:06:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-19T21:06:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:06:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-19T21:06:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:06:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:06:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:06:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-19T21:06:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:06:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:06:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:06:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:06:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:06:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:06:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_2 2026-02-19T21:06:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:06:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:06:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_2 2026-02-19T21:06:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-19T21:06:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:06:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-19T21:06:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:06:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:06:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:06:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:06:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ purge_snapshots cluster2 mirror image_2 2026-02-19T21:06:08.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:08.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:08.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:06:08.318 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2026-02-19T21:06:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-19T21:06:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:06:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:06:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:06:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_2 2026-02-19T21:06:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_2 2026-02-19T21:06:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-19T21:06:08.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-19T21:06:08.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T21:06:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:06:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T21:06:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_3 2026-02-19T21:06:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_3 512M 2026-02-19T21:06:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:06:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:06:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:06:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T21:06:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T21:06:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T21:06:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:06:08.509 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_3 2026-02-19T21:06:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ enable_mirror cluster2 mirror image_3 2026-02-19T21:06:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T21:06:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 journal 2026-02-19T21:06:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring enabled 2026-02-19T21:06:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2026-02-19T21:06:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_3': 2026-02-19T21:06:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T21:06:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:06:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 0 2026-02-19T21:06:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 34a0ba5b85f0 2026-02-19T21:06:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.34a0ba5b85f0 2026-02-19T21:06:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:06:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:06:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:06:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:06:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 21:06:08 2026 2026-02-19T21:06:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 21:06:08 2026 2026-02-19T21:06:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:06:08 2026 2026-02-19T21:06:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 34a0ba5b85f0 2026-02-19T21:06:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:06:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:06:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 614e1fac-bdbe-4887-a87c-35337da677f6 2026-02-19T21:06:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: true 2026-02-19T21:06:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-19T21:06:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:08.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T21:06:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=8 2026-02-19T21:06:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:06:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-19T21:06:10.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 58360832 bytes 2026-02-19T21:06:11.297 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 103725056 bytes 2026-02-19T21:06:12.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 186330112 bytes 2026-02-19T21:06:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 230085632 bytes 2026-02-19T21:06:13.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 273646592 bytes 2026-02-19T21:06:15.016 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-19T21:06:15.016 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T21:06:15.017 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-19T21:06:15.017 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-19T21:06:15.017 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-19T21:06:15.019 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-19T21:06:15.019 INFO:tasks.rbd_mirror_thrash:waiting for 89.60908505866068 secs before thrashing 2026-02-19T21:06:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 335381504 bytes 2026-02-19T21:06:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 359320576 bytes 2026-02-19T21:06:16.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T21:06:16.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:06:16.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T21:06:16.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:06:16.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2026-02-19T21:06:16.623 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=present 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_3 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-19T21:06:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:06:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=11fcf871cb67 2026-02-19T21:06:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:06:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:06:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_3 present 11fcf871cb67 2026-02-19T21:06:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:06:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=present 2026-02-19T21:06:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=11fcf871cb67 2026-02-19T21:06:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:06:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:06:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_3 2026-02-19T21:06:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:06:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:06:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T21:06:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-19T21:06:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:06:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=11fcf871cb67 2026-02-19T21:06:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 11fcf871cb67 2026-02-19T21:06:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 11fcf871cb67 -o 11fcf871cb67 = 11fcf871cb67 2026-02-19T21:06:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T21:06:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test present = present 2026-02-19T21:06:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:06:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ disable_mirror cluster2 mirror image_3 2026-02-19T21:06:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2026-02-19T21:06:16.891 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:06:16.886+0000 7ff7bdc78640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=614e1fac-bdbe-4887-a87c-35337da677f6 2026-02-19T21:06:43.355 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.38257861785362124) > weight (0.25) 2026-02-19T21:06:43.355 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-19T21:06:43.355 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-19T21:06:43.355 INFO:tasks.rbd_mirror_thrash:waiting for 1.3828490552637562 secs before reviving daemons 2026-02-19T21:06:43.356 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:06:43.351+0000 7ff720c3d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 120760) UID: 1000 2026-02-19T21:06:44.739 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-19T21:06:44.739 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T21:06:44.740 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-19T21:06:44.740 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-19T21:06:44.740 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-19T21:06:44.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-19T21:06:44.743 INFO:tasks.rbd_mirror_thrash:waiting for 85.17445815887888 secs before thrashing 2026-02-19T21:06:46.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring disabled 2026-02-19T21:06:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-19T21:06:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:06:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:06:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:06:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:06:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:06:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_3 2026-02-19T21:06:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:06:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:06:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T21:06:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-19T21:06:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:06:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-19T21:06:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:06:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:06:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:06:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-19T21:06:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:06:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:06:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:06:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:06:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:06:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_3 2026-02-19T21:06:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:06:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:06:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_3 2026-02-19T21:06:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-19T21:06:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:06:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-19T21:06:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:06:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:06:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:06:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:06:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ purge_snapshots cluster2 mirror image_3 2026-02-19T21:06:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2026-02-19T21:06:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-19T21:06:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:06:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:06:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_3 2026-02-19T21:06:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_3 2026-02-19T21:06:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-19T21:06:46.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-19T21:06:46.377 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T21:06:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:06:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T21:06:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_4 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_4 512M 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T21:06:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:06:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_4 2026-02-19T21:06:46.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ enable_mirror cluster2 mirror image_4 2026-02-19T21:06:46.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:46.419 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:46.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:06:46.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T21:06:46.420 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 journal 2026-02-19T21:06:46.453 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring enabled 2026-02-19T21:06:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2026-02-19T21:06:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_4': 2026-02-19T21:06:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 0 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 34c830ebb1fd 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.34c830ebb1fd 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 21:06:46 2026 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 21:06:46 2026 2026-02-19T21:06:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:06:46 2026 2026-02-19T21:06:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 34c830ebb1fd 2026-02-19T21:06:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:06:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:06:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: bbb55085-ae53-4004-bbd3-00e494fd839c 2026-02-19T21:06:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: true 2026-02-19T21:06:46.490 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-19T21:06:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:06:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T21:06:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=8 2026-02-19T21:06:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:06:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-19T21:06:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 29845504 bytes 2026-02-19T21:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 73607168 bytes 2026-02-19T21:06:48.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 142694400 bytes 2026-02-19T21:06:49.551 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 175762432 bytes 2026-02-19T21:06:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 213450752 bytes 2026-02-19T21:06:50.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 258992128 bytes 2026-02-19T21:06:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 314546176 bytes 2026-02-19T21:06:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 352844800 bytes 2026-02-19T21:06:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 355717120 bytes 2026-02-19T21:06:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 382121984 bytes 2026-02-19T21:06:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 427432960 bytes 2026-02-19T21:06:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 495736832 bytes 2026-02-19T21:06:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=present 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_4 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_4 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-19T21:06:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:06:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=11fca4b3e7d5 2026-02-19T21:06:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:06:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:06:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_4 present 11fca4b3e7d5 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=present 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=11fca4b3e7d5 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_4 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_4 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-19T21:06:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=11fca4b3e7d5 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 11fca4b3e7d5 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 11fca4b3e7d5 -o 11fca4b3e7d5 = 11fca4b3e7d5 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test present = present 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ disable_mirror cluster2 mirror image_4 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:06:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2026-02-19T21:06:54.845 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:06:54.831+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00414820 2026-02-19T21:06:54.845 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:06:54.831+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00e97d40 2026-02-19T21:06:54.845 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:06:54.831+0000 7fe4dd68c640 -1 reset not still connected to 0x564a00f1da00 2026-02-19T21:06:54.845 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:06:54.831+0000 7fe4dd68c640 -1 reset not still connected to 0x564a11738340 2026-02-19T21:06:54.845 INFO:tasks.ceph.cluster1.osd.2.trial013.stderr:2026-02-19T21:06:54.831+0000 7fe4dd68c640 -1 reset not still connected to 0x564a24146ea0 2026-02-19T21:07:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring disabled 2026-02-19T21:07:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-19T21:07:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:07:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:07:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:07:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:07:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:07:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:07:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_4 2026-02-19T21:07:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:07:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:07:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_4 2026-02-19T21:07:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-19T21:07:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:07:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-19T21:07:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:07:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:07:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:07:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-19T21:07:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:07:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:07:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:07:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:07:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:07:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:07:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_4 2026-02-19T21:07:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:07:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:07:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_4 2026-02-19T21:07:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-19T21:07:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:07:25.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-19T21:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ purge_snapshots cluster2 mirror image_4 2026-02-19T21:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2026-02-19T21:07:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-19T21:07:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:07:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:07:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:07:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:07:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_4 2026-02-19T21:07:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:07:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_4 2026-02-19T21:07:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-19T21:07:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-19T21:07:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T21:07:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:07:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T21:07:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_5 2026-02-19T21:07:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_5 512M 2026-02-19T21:07:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:07:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:07:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:07:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:07:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:07:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T21:07:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T21:07:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T21:07:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:07:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_5 2026-02-19T21:07:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ enable_mirror cluster2 mirror image_5 2026-02-19T21:07:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:07:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:07:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T21:07:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 journal 2026-02-19T21:07:25.466 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring enabled 2026-02-19T21:07:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2026-02-19T21:07:25.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_5': 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 0 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 34e0c6ef88cf 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.34e0c6ef88cf 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 21:07:25 2026 2026-02-19T21:07:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 21:07:25 2026 2026-02-19T21:07:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:07:25 2026 2026-02-19T21:07:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 34e0c6ef88cf 2026-02-19T21:07:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:07:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:07:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 29c4b276-a6d6-4a46-ae99-ba26acf88b13 2026-02-19T21:07:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: true 2026-02-19T21:07:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-19T21:07:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:07:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:07:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T21:07:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=7 2026-02-19T21:07:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:07:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-19T21:07:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 29600768 bytes 2026-02-19T21:07:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 54280192 bytes 2026-02-19T21:07:26.812 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T21:07:26.803+0000 7f5ce3652640 -1 reset not still connected to 0x56103bb5c5b0 2026-02-19T21:07:26.813 INFO:tasks.ceph.cluster2.osd.2.trial151.stderr:2026-02-19T21:07:26.803+0000 7f5ce3652640 -1 reset not still connected to 0x561042e3ac30 2026-02-19T21:07:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 64399360 bytes 2026-02-19T21:07:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 117307392 bytes 2026-02-19T21:07:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 154480640 bytes 2026-02-19T21:07:28.933 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 190344192 bytes 2026-02-19T21:07:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 213587968 bytes 2026-02-19T21:07:30.059 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 266517504 bytes 2026-02-19T21:07:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 328555520 bytes 2026-02-19T21:07:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 400331776 bytes 2026-02-19T21:07:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T21:07:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:07:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T21:07:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:07:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2026-02-19T21:07:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=present 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_5 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-19T21:07:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:07:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=11fcaeca5e1d 2026-02-19T21:07:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:07:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:07:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_5 present 11fcaeca5e1d 2026-02-19T21:07:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:07:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:07:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=present 2026-02-19T21:07:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=11fcaeca5e1d 2026-02-19T21:07:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:07:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:07:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_5 2026-02-19T21:07:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:07:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:07:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T21:07:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-19T21:07:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:07:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=11fcaeca5e1d 2026-02-19T21:07:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 11fcaeca5e1d 2026-02-19T21:07:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 11fcaeca5e1d -o 11fcaeca5e1d = 11fcaeca5e1d 2026-02-19T21:07:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T21:07:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test present = present 2026-02-19T21:07:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:07:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ disable_mirror cluster2 mirror image_5 2026-02-19T21:07:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:07:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:07:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:07:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2026-02-19T21:07:44.629 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.36981631194028675) > weight (0.25) 2026-02-19T21:07:44.629 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9425159034239035) > weight (0.25) 2026-02-19T21:07:44.629 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6087704987070996) > weight (0.25) 2026-02-19T21:07:44.629 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-19T21:07:44.630 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-19T21:07:44.630 INFO:tasks.rbd_mirror_thrash:waiting for 8.344211188131446 secs before reviving daemons 2026-02-19T21:07:44.630 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:07:44.623+0000 7fcb515f3640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 119337) UID: 1000 2026-02-19T21:07:52.974 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-19T21:07:52.975 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T21:07:52.975 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-19T21:07:52.975 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-19T21:07:52.975 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-19T21:07:52.978 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-19T21:07:52.978 INFO:tasks.rbd_mirror_thrash:waiting for 114.5229343692433 secs before thrashing 2026-02-19T21:08:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring disabled 2026-02-19T21:08:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-19T21:08:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:08:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:08:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:08:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:08:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:08:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:08:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_5 2026-02-19T21:08:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:08:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:08:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T21:08:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-19T21:08:03.279 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:08:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-19T21:08:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:08:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:08:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:08:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-19T21:08:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:08:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:08:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:08:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:08:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:08:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:08:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_5 2026-02-19T21:08:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:08:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:08:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_5 2026-02-19T21:08:03.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-19T21:08:03.407 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:08:03.630 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-19T21:08:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:08:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:08:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:08:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:08:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ purge_snapshots cluster2 mirror image_5 2026-02-19T21:08:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:08:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2026-02-19T21:08:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-19T21:08:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:08:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:08:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:08:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_5 2026-02-19T21:08:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_5 2026-02-19T21:08:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-19T21:08:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-19T21:08:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T21:08:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:08:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T21:08:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_6 2026-02-19T21:08:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_6 512M 2026-02-19T21:08:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:08:03.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:03.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:08:03.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:03.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:08:03.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T21:08:03.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T21:08:03.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T21:08:03.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:08:03.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_6 2026-02-19T21:08:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ enable_mirror cluster2 mirror image_6 2026-02-19T21:08:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T21:08:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 journal 2026-02-19T21:08:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring enabled 2026-02-19T21:08:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2026-02-19T21:08:03.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_6': 2026-02-19T21:08:03.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 0 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 3502d2e28b9f 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.3502d2e28b9f 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 21:08:03 2026 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 21:08:03 2026 2026-02-19T21:08:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:08:03 2026 2026-02-19T21:08:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 3502d2e28b9f 2026-02-19T21:08:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:08:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:08:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: d5f3fe53-5fc0-41ff-a6ed-8ebd16173219 2026-02-19T21:08:03.922 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: true 2026-02-19T21:08:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-19T21:08:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T21:08:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=6 2026-02-19T21:08:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:08:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-19T21:08:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 26509312 bytes 2026-02-19T21:08:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 88665088 bytes 2026-02-19T21:08:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 110945280 bytes 2026-02-19T21:08:07.528 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 184856576 bytes 2026-02-19T21:08:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 208145408 bytes 2026-02-19T21:08:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 250408960 bytes 2026-02-19T21:08:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 263432192 bytes 2026-02-19T21:08:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 268780544 bytes 2026-02-19T21:08:09.588 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 302204928 bytes 2026-02-19T21:08:09.918 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.2575270561244465) > weight (0.25) 2026-02-19T21:08:09.918 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5080917619221087) > weight (0.25) 2026-02-19T21:08:09.918 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9715295938782668) > weight (0.25) 2026-02-19T21:08:09.918 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.5893871409616296) > weight (0.25) 2026-02-19T21:08:09.918 INFO:tasks.rbd_mirror_thrash:waiting for 80.53419873713548 secs before thrashing 2026-02-19T21:08:09.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T21:08:09.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:08:09.936 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=present 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_6 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:08:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:08:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T21:08:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-19T21:08:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:08:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=11fc96058aa 2026-02-19T21:08:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:08:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:08:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_6 present 11fc96058aa 2026-02-19T21:08:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:08:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=present 2026-02-19T21:08:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=11fc96058aa 2026-02-19T21:08:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:08:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:08:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_6 2026-02-19T21:08:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:08:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:08:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T21:08:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-19T21:08:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:08:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=11fc96058aa 2026-02-19T21:08:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 11fc96058aa 2026-02-19T21:08:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 11fc96058aa -o 11fc96058aa = 11fc96058aa 2026-02-19T21:08:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T21:08:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test present = present 2026-02-19T21:08:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:08:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ disable_mirror cluster2 mirror image_6 2026-02-19T21:08:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2026-02-19T21:08:40.063 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring disabled 2026-02-19T21:08:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-19T21:08:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_6 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T21:08:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-19T21:08:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:08:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-19T21:08:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:08:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:08:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:08:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-19T21:08:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:08:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_6 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_6 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-19T21:08:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:08:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-19T21:08:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:08:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:08:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:08:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:08:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ purge_snapshots cluster2 mirror image_6 2026-02-19T21:08:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2026-02-19T21:08:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-19T21:08:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:08:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:08:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_6 2026-02-19T21:08:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_6 2026-02-19T21:08:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-19T21:08:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-19T21:08:40.613 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_7 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_7 512M 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:08:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_7 2026-02-19T21:08:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ enable_mirror cluster2 mirror image_7 2026-02-19T21:08:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:08:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T21:08:40.656 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 journal 2026-02-19T21:08:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring enabled 2026-02-19T21:08:40.691 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2026-02-19T21:08:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_7': 2026-02-19T21:08:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T21:08:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:08:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 0 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 351aca454b3c 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.351aca454b3c 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 21:08:40 2026 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 21:08:40 2026 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:08:40 2026 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 351aca454b3c 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:08:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:08:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 57de7c02-f269-4692-81ce-da761e93234b 2026-02-19T21:08:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: true 2026-02-19T21:08:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-19T21:08:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:40.720 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:08:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T21:08:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=6 2026-02-19T21:08:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:08:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-19T21:08:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 22365184 bytes 2026-02-19T21:08:42.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 108883968 bytes 2026-02-19T21:08:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 155183104 bytes 2026-02-19T21:08:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 183332864 bytes 2026-02-19T21:08:45.161 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 203719680 bytes 2026-02-19T21:08:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 207065088 bytes 2026-02-19T21:08:45.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 238251008 bytes 2026-02-19T21:08:46.373 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 305694720 bytes 2026-02-19T21:08:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T21:08:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:08:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=present 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_7 2026-02-19T21:08:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:08:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:08:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T21:08:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-19T21:08:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:08:46.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=11fc56f73a3b 2026-02-19T21:08:46.965 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:08:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:08:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_7 present 11fc56f73a3b 2026-02-19T21:08:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:08:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=present 2026-02-19T21:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=11fc56f73a3b 2026-02-19T21:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_7 2026-02-19T21:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T21:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-19T21:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:08:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=11fc56f73a3b 2026-02-19T21:08:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 11fc56f73a3b 2026-02-19T21:08:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 11fc56f73a3b -o 11fc56f73a3b = 11fc56f73a3b 2026-02-19T21:08:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T21:08:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test present = present 2026-02-19T21:08:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:08:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ disable_mirror cluster2 mirror image_7 2026-02-19T21:08:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:08:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:08:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:08:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2026-02-19T21:09:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring disabled 2026-02-19T21:09:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-19T21:09:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:09:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:09:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:09:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:09:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:09:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:09:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_7 2026-02-19T21:09:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:09:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:09:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T21:09:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-19T21:09:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:09:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-19T21:09:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:09:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:09:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:09:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-19T21:09:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:09:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:17.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:09:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:09:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:09:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:09:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:09:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_7 2026-02-19T21:09:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:09:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:09:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_7 2026-02-19T21:09:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-19T21:09:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:09:17.258 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-19T21:09:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:09:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:09:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:09:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:09:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ purge_snapshots cluster2 mirror image_7 2026-02-19T21:09:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:09:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:09:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2026-02-19T21:09:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-19T21:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:09:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_7 2026-02-19T21:09:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:09:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_7 2026-02-19T21:09:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-19T21:09:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-19T21:09:17.457 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T21:09:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:09:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T21:09:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_8 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_8 512M 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T21:09:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:09:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_8 2026-02-19T21:09:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ enable_mirror cluster2 mirror image_8 2026-02-19T21:09:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:09:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:09:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T21:09:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 journal 2026-02-19T21:09:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring enabled 2026-02-19T21:09:17.537 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2026-02-19T21:09:17.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_8': 2026-02-19T21:09:17.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T21:09:17.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:09:17.566 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 0 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 3532e333e050 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.3532e333e050 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 21:09:17 2026 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 21:09:17 2026 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:09:17 2026 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 3532e333e050 2026-02-19T21:09:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:09:17.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:09:17.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: a1a7e495-6c70-4bf7-bbe0-2dc1b8014acf 2026-02-19T21:09:17.568 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: true 2026-02-19T21:09:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-19T21:09:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:09:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:09:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T21:09:17.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=12 2026-02-19T21:09:17.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:09:17.573 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-19T21:09:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 37184512 bytes 2026-02-19T21:09:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 74855424 bytes 2026-02-19T21:09:19.217 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 92194816 bytes 2026-02-19T21:09:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 168832000 bytes 2026-02-19T21:09:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 216240128 bytes 2026-02-19T21:09:22.090 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 280410112 bytes 2026-02-19T21:09:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 300528640 bytes 2026-02-19T21:09:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 332184576 bytes 2026-02-19T21:09:23.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 356224000 bytes 2026-02-19T21:09:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 391443456 bytes 2026-02-19T21:09:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 485406720 bytes 2026-02-19T21:09:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 507798528 bytes 2026-02-19T21:09:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 539384832 bytes 2026-02-19T21:09:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 560313344 bytes 2026-02-19T21:09:26.843 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 603254784 bytes 2026-02-19T21:09:27.454 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 642954240 bytes 2026-02-19T21:09:28.635 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 706438144 bytes 2026-02-19T21:09:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 754320384 bytes 2026-02-19T21:09:29.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T21:09:29.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:09:29.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T21:09:29.585 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=present 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_8 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:09:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T21:09:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-19T21:09:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:09:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=11fc8ed456f5 2026-02-19T21:09:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:09:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:09:29.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_8 present 11fc8ed456f5 2026-02-19T21:09:29.721 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=present 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=11fc8ed456f5 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_8 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-19T21:09:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:09:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=11fc8ed456f5 2026-02-19T21:09:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 11fc8ed456f5 2026-02-19T21:09:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 11fc8ed456f5 -o 11fc8ed456f5 = 11fc8ed456f5 2026-02-19T21:09:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T21:09:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test present = present 2026-02-19T21:09:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:09:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ disable_mirror cluster2 mirror image_8 2026-02-19T21:09:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:09:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:09:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:09:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2026-02-19T21:09:30.453 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5495495807967565) > weight (0.25) 2026-02-19T21:09:30.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7141996729005338) > weight (0.25) 2026-02-19T21:09:30.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8930090070212487) > weight (0.25) 2026-02-19T21:09:30.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.2632458209110179) > weight (0.25) 2026-02-19T21:09:30.454 INFO:tasks.rbd_mirror_thrash:waiting for 102.06619040533351 secs before thrashing 2026-02-19T21:09:47.502 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8988120432280324) > weight (0.25) 2026-02-19T21:09:47.502 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6721193087127498) > weight (0.25) 2026-02-19T21:09:47.502 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-19T21:09:47.503 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-19T21:09:47.503 INFO:tasks.rbd_mirror_thrash:waiting for 1.7260658062385525 secs before reviving daemons 2026-02-19T21:09:47.503 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:09:47.499+0000 7f4032974640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 114119) UID: 1000 2026-02-19T21:09:49.230 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-19T21:09:49.230 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T21:09:49.230 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-19T21:09:49.230 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-19T21:09:49.231 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-19T21:09:49.233 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-19T21:09:49.233 INFO:tasks.rbd_mirror_thrash:waiting for 89.92483670124504 secs before thrashing 2026-02-19T21:10:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring disabled 2026-02-19T21:10:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-19T21:10:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:10:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:10:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:10:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:10:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:10:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:10:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_8 2026-02-19T21:10:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:10:00.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:10:00.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T21:10:00.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-19T21:10:00.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:10:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-19T21:10:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:10:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:10:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:10:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-19T21:10:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:10:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:10:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:10:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:10:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:10:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:10:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_8 2026-02-19T21:10:00.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:10:00.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:10:00.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_8 2026-02-19T21:10:00.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-19T21:10:00.253 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:10:00.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-19T21:10:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:10:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:10:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:10:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:10:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ purge_snapshots cluster2 mirror image_8 2026-02-19T21:10:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:10:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2026-02-19T21:10:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-19T21:10:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:10:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:10:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:10:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_8 2026-02-19T21:10:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_8 2026-02-19T21:10:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-19T21:10:00.346 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-19T21:10:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T21:10:00.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:10:00.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T21:10:00.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_9 2026-02-19T21:10:00.484 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_9 512M 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:10:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_9 2026-02-19T21:10:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ enable_mirror cluster2 mirror image_9 2026-02-19T21:10:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T21:10:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 journal 2026-02-19T21:10:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring enabled 2026-02-19T21:10:00.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2026-02-19T21:10:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_9': 2026-02-19T21:10:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T21:10:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:10:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 0 2026-02-19T21:10:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 35536e2aae1d 2026-02-19T21:10:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.35536e2aae1d 2026-02-19T21:10:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:10:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:10:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:10:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:10:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 21:10:00 2026 2026-02-19T21:10:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 21:10:00 2026 2026-02-19T21:10:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:10:00 2026 2026-02-19T21:10:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 35536e2aae1d 2026-02-19T21:10:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:10:00.581 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:10:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 1734a089-60fa-4510-9d87-19689854a981 2026-02-19T21:10:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: true 2026-02-19T21:10:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-19T21:10:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T21:10:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=13 2026-02-19T21:10:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:10:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-19T21:10:00.756 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 19686400 bytes 2026-02-19T21:10:01.681 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 68722688 bytes 2026-02-19T21:10:02.308 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 106532864 bytes 2026-02-19T21:10:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 138729472 bytes 2026-02-19T21:10:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 150936576 bytes 2026-02-19T21:10:03.380 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 179083264 bytes 2026-02-19T21:10:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 226628608 bytes 2026-02-19T21:10:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 259747840 bytes 2026-02-19T21:10:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 298314752 bytes 2026-02-19T21:10:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1000 IOs, 349210624 bytes 2026-02-19T21:10:06.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1100 IOs, 368489472 bytes 2026-02-19T21:10:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1200 IOs, 406192128 bytes 2026-02-19T21:10:07.618 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1300 IOs, 440395776 bytes 2026-02-19T21:10:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1400 IOs, 525471744 bytes 2026-02-19T21:10:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1500 IOs, 526108672 bytes 2026-02-19T21:10:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1600 IOs, 560917504 bytes 2026-02-19T21:10:09.663 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1700 IOs, 584150016 bytes 2026-02-19T21:10:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1800 IOs, 619128832 bytes 2026-02-19T21:10:10.897 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:1900 IOs, 675964928 bytes 2026-02-19T21:10:12.040 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2000 IOs, 787782656 bytes 2026-02-19T21:10:13.250 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:2100 IOs, 855219200 bytes 2026-02-19T21:10:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T21:10:13.596 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=present 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:10:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_9 2026-02-19T21:10:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:10:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:10:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T21:10:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-19T21:10:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:10:13.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=11fc967fd151 2026-02-19T21:10:13.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:10:13.627 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_9 present 11fc967fd151 2026-02-19T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=present 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=11fc967fd151 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_9 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-19T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:10:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=11fc967fd151 2026-02-19T21:10:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 11fc967fd151 2026-02-19T21:10:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 11fc967fd151 -o 11fc967fd151 = 11fc967fd151 2026-02-19T21:10:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T21:10:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test present = present 2026-02-19T21:10:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:10:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ disable_mirror cluster2 mirror image_9 2026-02-19T21:10:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2026-02-19T21:10:13.853 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:10:13.843+0000 7ff7bd477640 -1 rbd::mirror::ImageMap: 0x564fec9bc7e0 handle_peer_ack_remove: failed to remove global_image_id=1734a089-60fa-4510-9d87-19689854a981 2026-02-19T21:10:44.144 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring disabled 2026-02-19T21:10:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-19T21:10:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:10:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:10:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:10:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:10:44.152 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:10:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_9 2026-02-19T21:10:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:10:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:10:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T21:10:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-19T21:10:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:10:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-19T21:10:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:10:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:10:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:10:44.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-19T21:10:44.485 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_9 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_9 2026-02-19T21:10:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-19T21:10:44.487 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:10:44.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-19T21:10:44.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:10:44.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:10:44.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:10:44.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:10:44.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ purge_snapshots cluster2 mirror image_9 2026-02-19T21:10:44.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:44.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:44.510 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:44.511 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2026-02-19T21:10:44.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-19T21:10:44.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:44.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:44.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:44.540 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:10:44.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:10:44.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_9 2026-02-19T21:10:44.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:44.541 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:44.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_9 2026-02-19T21:10:44.542 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-19T21:10:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-19T21:10:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T21:10:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:10:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for i in `seq 1 10` 2026-02-19T21:10:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image=image_10 2026-02-19T21:10:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ create_image cluster2 mirror image_10 512M 2026-02-19T21:10:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:10:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:10:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:10:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:10:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local size=128 2026-02-19T21:10:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -n 512M ']' 2026-02-19T21:10:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ size=512M 2026-02-19T21:10:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ shift 2026-02-19T21:10:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_10 2026-02-19T21:10:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ enable_mirror cluster2 mirror image_10 2026-02-19T21:10:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:10:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local mode=journal 2026-02-19T21:10:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 journal 2026-02-19T21:10:44.784 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring enabled 2026-02-19T21:10:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:rbd image 'image_10': 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: size 512 MiB in 128 objects 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: order 22 (4 MiB objects) 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: snapshot_count: 0 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: id: 356bb63e622c 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: block_name_prefix: rbd_data.356bb63e622c 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: format: 2 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: features: layering, exclusive-lock, journaling 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: op_features: 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: flags: 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: create_timestamp: Thu Feb 19 21:10:44 2026 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: access_timestamp: Thu Feb 19 21:10:44 2026 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: modify_timestamp: Thu Feb 19 21:10:44 2026 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: journal: 356bb63e622c 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring state: enabled 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring mode: journal 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring global id: 3ed0b1f5-3139-43cf-a2a2-d9add7ba5396 2026-02-19T21:10:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout: mirroring primary: true 2026-02-19T21:10:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-19T21:10:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:10:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-19T21:10:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local duration=7 2026-02-19T21:10:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:10:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-19T21:10:45.350 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:100 IOs, 29782016 bytes 2026-02-19T21:10:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:200 IOs, 70411264 bytes 2026-02-19T21:10:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:300 IOs, 97295360 bytes 2026-02-19T21:10:46.807 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:400 IOs, 126331904 bytes 2026-02-19T21:10:48.230 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:500 IOs, 167367680 bytes 2026-02-19T21:10:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:600 IOs, 209449984 bytes 2026-02-19T21:10:49.184 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:700 IOs, 225677312 bytes 2026-02-19T21:10:50.743 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:800 IOs, 315329536 bytes 2026-02-19T21:10:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:900 IOs, 344951808 bytes 2026-02-19T21:10:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=124 2026-02-19T21:10:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:10:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 124 -eq 124 ']' 2026-02-19T21:10:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:10:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2026-02-19T21:10:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:10:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:10:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=present 2026-02-19T21:10:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:10:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:10:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:10:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_10 2026-02-19T21:10:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:10:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:10:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T21:10:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-19T21:10:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:10:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id=11fccbcbdf0e 2026-02-19T21:10:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:10:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:10:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_10 present 11fccbcbdf0e 2026-02-19T21:10:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:10:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:10:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=present 2026-02-19T21:10:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id=11fccbcbdf0e 2026-02-19T21:10:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:10:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:10:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_10 2026-02-19T21:10:51.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:10:51.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:10:51.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T21:10:51.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-19T21:10:51.971 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:10:52.001 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id=11fccbcbdf0e 2026-02-19T21:10:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 11fccbcbdf0e 2026-02-19T21:10:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -z 11fccbcbdf0e -o 11fccbcbdf0e = 11fccbcbdf0e 2026-02-19T21:10:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_state=present 2026-02-19T21:10:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test present = present 2026-02-19T21:10:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:10:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ disable_mirror cluster2 mirror image_10 2026-02-19T21:10:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:10:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:10:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:10:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2026-02-19T21:11:12.521 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-19T21:11:12.522 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-19T21:11:12.522 INFO:tasks.rbd_mirror_thrash:waiting for 2.9194400608051527 secs before reviving daemons 2026-02-19T21:11:12.523 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:12.519+0000 7f654dfc8640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 115623) UID: 1000 2026-02-19T21:11:15.442 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-19T21:11:15.443 INFO:teuthology.orchestra.run:waiting for 600 2026-02-19T21:11:15.443 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-19T21:11:15.443 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-19T21:11:15.443 DEBUG:teuthology.orchestra.run.trial151:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-19T21:11:15.445 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-19T21:11:15.446 INFO:tasks.rbd_mirror_thrash:waiting for 71.68296661650417 secs before thrashing 2026-02-19T21:11:19.159 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7503862291296896) > weight (0.25) 2026-02-19T21:11:19.159 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4879252755262973) > weight (0.25) 2026-02-19T21:11:19.159 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6992503377681454) > weight (0.25) 2026-02-19T21:11:19.160 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.7973248939446849) > weight (0.25) 2026-02-19T21:11:19.160 INFO:tasks.rbd_mirror_thrash:waiting for 75.4120677745106 secs before thrashing 2026-02-19T21:11:22.073 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:Mirroring disabled 2026-02-19T21:11:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-19T21:11:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:11:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:11:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local state=deleted 2026-02-19T21:11:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:11:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local s 2026-02-19T21:11:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:11:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_10 2026-02-19T21:11:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:11:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T21:11:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-19T21:11:22.084 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:11:22.108 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-19T21:11:22.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ image_id= 2026-02-19T21:11:22.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-19T21:11:22.112 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0.1 2026-02-19T21:11:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-19T21:11:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:11:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:11:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local test_state=deleted 2026-02-19T21:11:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image_id= 2026-02-19T21:11:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_state=deleted 2026-02-19T21:11:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local current_image_id 2026-02-19T21:11:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ get_image_id cluster1 mirror image_10 2026-02-19T21:11:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:11:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local image=image_10 2026-02-19T21:11:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-19T21:11:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-19T21:11:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-19T21:11:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ current_image_id= 2026-02-19T21:11:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n '' 2026-02-19T21:11:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test deleted = deleted 2026-02-19T21:11:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:11:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ purge_snapshots cluster2 mirror image_10 2026-02-19T21:11:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:11:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:11:22.242 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster cluster2 snap purge mirror/image_10 2026-02-19T21:11:22.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-19T21:11:22.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:11:22.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:11:22.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-19T21:11:22.274 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:11:22.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ remove_image cluster2 mirror image_10 2026-02-19T21:11:22.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:11:22.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local image=image_10 2026-02-19T21:11:22.275 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-19T21:11:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-19T21:11:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-19T21:11:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:11:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-02-19T21:11:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ tee -a /tmp/tmp.xpxdoJdHT7/rbd-mirror.test.log 2026-02-19T21:11:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ date '+%F %T' 2026-02-19T21:11:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo 2026-02-19 21:11:22 TEST: check if removed 'images'\''' OMAP are removed 2026-02-19T21:11:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:2026-02-19 21:11:22 TEST: check if removed images' OMAP are removed 2026-02-19T21:11:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-19T21:11:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:11:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-19T21:11:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:11:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local obj_name=rbd_mirroring 2026-02-19T21:11:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local filter=status_global 2026-02-19T21:11:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-19T21:11:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:11:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:11:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-19T21:11:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:11:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirroring 2026-02-19T21:11:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local filter=status_global 2026-02-19T21:11:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-19T21:11:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:11:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -c status_global 2026-02-19T21:11:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirroring 2026-02-19T21:11:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 = 0 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=0 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 0 ']' 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local obj_name=rbd_mirroring 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local filter=image_ 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:11:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:11:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-19T21:11:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:11:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirroring 2026-02-19T21:11:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local filter=image_ 2026-02-19T21:11:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-19T21:11:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:11:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirroring 2026-02-19T21:11:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -c image_ 2026-02-19T21:11:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-19T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 = 0 2026-02-19T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=0 2026-02-19T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 0 ']' 2026-02-19T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-19T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local obj_name=rbd_mirror_leader 2026-02-19T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local filter=image_map 2026-02-19T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-19T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:11:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-19T21:11:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:11:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirror_leader 2026-02-19T21:11:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local filter=image_map 2026-02-19T21:11:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-19T21:11:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster1 2026-02-19T21:11:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -c image_map 2026-02-19T21:11:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirror_leader 2026-02-19T21:11:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-19T21:11:22.530 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 = 0 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=0 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 0 ']' 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local obj_name=rbd_mirroring 2026-02-19T21:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local filter=status_global 2026-02-19T21:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-19T21:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:11:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:11:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-19T21:11:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:11:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirroring 2026-02-19T21:11:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local filter=status_global 2026-02-19T21:11:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-19T21:11:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:11:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirroring 2026-02-19T21:11:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -c status_global 2026-02-19T21:11:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-19T21:11:22.552 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 = 0 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=0 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 0 ']' 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local obj_name=rbd_mirroring 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local filter=image_ 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-19T21:11:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:11:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:11:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-19T21:11:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:11:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirroring 2026-02-19T21:11:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local filter=image_ 2026-02-19T21:11:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-19T21:11:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -c image_ 2026-02-19T21:11:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:11:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirroring 2026-02-19T21:11:22.556 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-19T21:11:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 = 0 2026-02-19T21:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=0 2026-02-19T21:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 0 ']' 2026-02-19T21:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-19T21:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local pool=mirror 2026-02-19T21:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local obj_name=rbd_mirror_leader 2026-02-19T21:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local filter=image_map 2026-02-19T21:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-19T21:11:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ sleep 0 2026-02-19T21:11:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:11:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-19T21:11:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:11:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirror_leader 2026-02-19T21:11:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local filter=image_map 2026-02-19T21:11:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-19T21:11:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local cluster=cluster2 2026-02-19T21:11:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local pool=mirror 2026-02-19T21:11:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ local obj_name=rbd_mirror_leader 2026-02-19T21:11:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-19T21:11:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ grep -c image_map 2026-02-19T21:11:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test 0 = 0 2026-02-19T21:11:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ error_code=0 2026-02-19T21:11:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set -e 2026-02-19T21:11:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 0 ']' 2026-02-19T21:11:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 0 2026-02-19T21:11:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ cleanup 0 2026-02-19T21:11:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local error_code=0 2026-02-19T21:11:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ set +e 2026-02-19T21:11:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -ne 0 ']' 2026-02-19T21:11:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -z '' ']' 2026-02-19T21:11:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster instance 2026-02-19T21:11:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T21:11:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-19T21:11:22.906 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:pool 'mirror' does not exist 2026-02-19T21:11:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T21:11:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-19T21:11:22.942 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:22.931+0000 7faaab268640 -1 rbd::mirror::LeaderWatcher: 0x55fa57972000 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-19T21:11:22.942 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:22.931+0000 7faaa4a5b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55fa587c0d80 handle_unregister_watcher: error unregistering watcher for trash directory: (107) Transport endpoint is not connected 2026-02-19T21:11:22.942 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:22.931+0000 7faaa4a5b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55fa581778c0 handle_unregister_watcher: error unregistering watcher for trash directory: (107) Transport endpoint is not connected 2026-02-19T21:11:22.942 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:22.931+0000 7faaa4a5b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55fa581766c0 handle_unregister_watcher: error unregistering watcher for trash directory: (107) Transport endpoint is not connected 2026-02-19T21:11:22.942 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:22.931+0000 7faaa525c640 -1 rbd::mirror::PoolWatcher: 0x55fa580c6780 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:22.942 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:22.935+0000 7faaa4a5b640 -1 rbd::mirror::PoolWatcher: 0x55fa587f9e00 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:22.942 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:22.935+0000 7faaa525c640 -1 rbd::mirror::PoolWatcher: 0x55fa585dc5a0 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:22.945 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:22.935+0000 7faaab268640 -1 rbd::mirror::LeaderWatcher: 0x55fa57972000 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-19T21:11:22.945 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:22.935+0000 7faaab268640 -1 rbd::mirror::LeaderWatcher: 0x55fa57972000 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-19T21:11:23.083 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.075+0000 7f890161f640 -1 rbd::mirror::RemotePoolPoller: 0x55c5d47fbee0 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2026-02-19T21:11:23.083 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.075+0000 7f8900e1e640 -1 rbd::mirror::RemotePoolPoller: 0x55c5d47fbee0 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2026-02-19T21:11:23.083 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.075+0000 7f8900e1e640 -1 rbd::mirror::PoolReplayer: 0x55c5d480b800 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2026-02-19T21:11:23.091 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.083+0000 7f890161f640 -1 rbd::mirror::MirrorStatusUpdater 0x55c5d598a5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:23.091 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.083+0000 7f890e639640 -1 rbd::mirror::NamespaceReplayer: 0x55c5d4811340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:23.092 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.083+0000 7f8900e1e640 -1 rbd::mirror::MirrorStatusUpdater 0x55c5d598a480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:23.092 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.083+0000 7f890e639640 -1 rbd::mirror::NamespaceReplayer: 0x55c5d4810a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:23.092 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.083+0000 7f890161f640 -1 rbd::mirror::MirrorStatusUpdater 0x55c5d598a360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:23.092 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.083+0000 7f890e639640 -1 rbd::mirror::NamespaceReplayer: 0x55c5d4810380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:23.160 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.151+0000 7f8911e7f980 -1 rbd::mirror::Mirror: 0x55c5d4872360 update_pool_replayers: restarting failed pool replayer for uuid: 63a6310b-64c5-44db-b9f4-d0f0211d79bf cluster: cluster1 client: client.mirror 2026-02-19T21:11:23.185 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:23.175+0000 7f8911e7f980 -1 rbd::mirror::PoolReplayer: 0x55c5d480b800 init: error accessing remote pool mirror: (2) No such file or directory 2026-02-19T21:11:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:pool 'mirror' does not exist 2026-02-19T21:11:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T21:11:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-19T21:11:24.900 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:24.891+0000 7faaab268640 -1 rbd::mirror::LeaderWatcher: 0x55fa57f64900 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-19T21:11:24.900 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:24.891+0000 7faa97a41640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55fa5d1e0fc0 handle_unregister_watcher: error unregistering watcher for trash directory: (107) Transport endpoint is not connected 2026-02-19T21:11:24.900 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:24.891+0000 7faa97a41640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55fa58f8b8c0 handle_unregister_watcher: error unregistering watcher for trash directory: (107) Transport endpoint is not connected 2026-02-19T21:11:24.900 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:24.891+0000 7faa97240640 -1 rbd::mirror::PoolWatcher: 0x55fa5b48ba40 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:24.900 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:24.891+0000 7faa97a41640 -1 rbd::mirror::PoolWatcher: 0x55fa5d0d0000 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:24.917 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:pool 'mirror_parent' does not exist 2026-02-19T21:11:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ CEPH_ARGS= 2026-02-19T21:11:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-19T21:11:24.939 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:24.931+0000 7faaab268640 -1 rbd::mirror::LeaderWatcher: 0x55fa57f64900 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-19T21:11:24.939 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:24.931+0000 7faaab268640 -1 rbd::mirror::LeaderWatcher: 0x55fa57f64900 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-19T21:11:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:pool 'mirror_parent' does not exist 2026-02-19T21:11:26.018 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-19T21:11:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stop_mirrors cluster1 2026-02-19T21:11:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1 2026-02-19T21:11:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local sig= 2026-02-19T21:11:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:11:26.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:11:26.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stop_mirror cluster1:0 '' 2026-02-19T21:11:26.021 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:0 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local sig= 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stop_mirror cluster1:1 '' 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:1 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local sig= 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stop_mirror cluster1:2 '' 2026-02-19T21:11:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:2 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local sig= 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stop_mirror cluster1:3 '' 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster1:3 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local sig= 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stop_mirrors cluster2 2026-02-19T21:11:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2 2026-02-19T21:11:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local sig= 2026-02-19T21:11:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:++ seq 0 3 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stop_mirror cluster2:0 '' 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2:0 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local sig= 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stop_mirror cluster2:1 '' 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2:1 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local sig= 2026-02-19T21:11:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stop_mirror cluster2:2 '' 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2:2 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local sig= 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ stop_mirror cluster2:3 '' 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local cluster=cluster2:3 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ local sig= 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test -n 1 2026-02-19T21:11:26.027 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ return 2026-02-19T21:11:26.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' -z 1 ']' 2026-02-19T21:11:26.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ test '' = /tmp/tmp.xpxdoJdHT7 2026-02-19T21:11:26.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ rm -Rf /tmp/tmp.xpxdoJdHT7 2026-02-19T21:11:26.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ '[' 0 -eq 0 ']' 2026-02-19T21:11:26.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ echo OK 2026-02-19T21:11:26.028 INFO:tasks.workunit.cluster1.client.mirror.trial151.stdout:OK 2026-02-19T21:11:26.029 INFO:tasks.workunit.cluster1.client.mirror.trial151.stderr:+ exit 0 2026-02-19T21:11:26.030 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-19T21:11:26.030 DEBUG:teuthology.orchestra.run.trial151:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-19T21:11:26.083 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_stress.sh'] on cluster1.client.mirror... 2026-02-19T21:11:26.084 DEBUG:teuthology.orchestra.run.trial151:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-19T21:11:26.391 DEBUG:teuthology.parallel:result is None 2026-02-19T21:11:26.392 DEBUG:teuthology.orchestra.run.trial151:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-19T21:11:26.401 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-19T21:11:26.402 DEBUG:teuthology.orchestra.run.trial151:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-19T21:11:26.449 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-19T21:11:26.449 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-19T21:11:26.468 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-19T21:11:26.468 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 16 2026-02-19T21:11:26.468 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-19T21:11:26.468 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-19T21:11:26.485 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-19T21:11:26.485 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 22 2026-02-19T21:11:26.485 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-19T21:11:26.485 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-19T21:11:26.502 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.3:waiting for process to exit 2026-02-19T21:11:26.502 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:11:26.504 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f890ee3a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 126470) UID: 1000 2026-02-19T21:11:26.504 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f890de38640 -1 rbd::mirror::InstanceWatcher: 0x55c5d5dfc380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13558 object: (2) No such file or directory 2026-02-19T21:11:26.504 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f890de38640 -1 rbd::mirror::InstanceWatcher: 0x55c5d5dfca80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13558 object: (2) No such file or directory 2026-02-19T21:11:26.504 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f88fa611640 -1 rbd::mirror::InstanceWatcher: 0x55c5d5dfc380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.504 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f88fa611640 -1 rbd::mirror::InstanceWatcher: 0x55c5d5dfca80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.504 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f88f3e04640 -1 rbd::mirror::MirrorStatusUpdater 0x55c5d5be5320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.504 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f88f3e04640 -1 rbd::mirror::MirrorStatusUpdater 0x55c5d598a900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.505 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f890e639640 -1 rbd::mirror::NamespaceReplayer: 0x55c5d5ade380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.505 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f890e639640 -1 rbd::mirror::NamespaceReplayer: 0x55c5d5ade540 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.505 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f88f9e10640 -1 rbd::mirror::MirrorStatusUpdater 0x55c5d5be4fc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.505 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f88f9e10640 -1 rbd::mirror::MirrorStatusUpdater 0x55c5d5be50e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.505 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f890de38640 -1 rbd::mirror::NamespaceReplayer: 0x55c5d5ade380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.505 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f890de38640 -1 rbd::mirror::NamespaceReplayer: 0x55c5d5ade540 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.505 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.499+0000 7f890e639640 -1 rbd::mirror::LeaderWatcher: 0x55c5d5ea2900 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.519 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Stopped 2026-02-19T21:11:26.519 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-19T21:11:26.537 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.2:waiting for process to exit 2026-02-19T21:11:26.537 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:11:26.565 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a9bf9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 127427) UID: 1000 2026-02-19T21:11:26.567 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a8bf7640 -1 rbd::mirror::InstanceWatcher: 0x55e41e316700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13637 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.568 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a93f8640 -1 rbd::mirror::InstanceWatcher: 0x55e41e5dc380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13639 object: (2) No such file or directory 2026-02-19T21:11:26.568 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a93f8640 -1 rbd::mirror::InstanceWatcher: 0x55e41e316a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13639 object: (2) No such file or directory 2026-02-19T21:11:26.568 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a2beb640 -1 rbd::mirror::InstanceWatcher: 0x55e41e316000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.568 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a23ea640 -1 rbd::mirror::InstanceWatcher: 0x55e41e316380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.568 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f629bbdd640 -1 rbd::mirror::MirrorStatusUpdater 0x55e41e146360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.568 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a2beb640 -1 rbd::mirror::InstanceWatcher: 0x55e41e316700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.568 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a8bf7640 -1 rbd::mirror::NamespaceReplayer: 0x55e41cde1180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.568 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f629c3de640 -1 rbd::mirror::MirrorStatusUpdater 0x55e41e146480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.568 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f629c3de640 -1 rbd::mirror::MirrorStatusUpdater 0x55e41e1465a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.568 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f6294bcf640 -1 rbd::mirror::InstanceWatcher: 0x55e41e5dc380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a93f8640 -1 rbd::mirror::NamespaceReplayer: 0x55e41cde0a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f6294bcf640 -1 rbd::mirror::InstanceWatcher: 0x55e41e316a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a93f8640 -1 rbd::mirror::NamespaceReplayer: 0x55e41cde0380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f628ebc3640 -1 rbd::mirror::MirrorStatusUpdater 0x55e41e3cd440 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f628ebc3640 -1 rbd::mirror::MirrorStatusUpdater 0x55e41e3cd0e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a8bf7640 -1 rbd::mirror::NamespaceReplayer: 0x55e41e240a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a8bf7640 -1 rbd::mirror::NamespaceReplayer: 0x55e41e240c40 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62953d0640 -1 rbd::mirror::MirrorStatusUpdater 0x55e41e3ccd80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62953d0640 -1 rbd::mirror::MirrorStatusUpdater 0x55e41e3ccea0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a93f8640 -1 rbd::mirror::NamespaceReplayer: 0x55e41e240a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a93f8640 -1 rbd::mirror::NamespaceReplayer: 0x55e41e240c40 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.559+0000 7f62a93f8640 -1 rbd::mirror::LeaderWatcher: 0x55e41e664900 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.582 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Stopped 2026-02-19T21:11:26.582 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-19T21:11:26.600 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.1:waiting for process to exit 2026-02-19T21:11:26.601 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:11:26.629 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faac3337640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 125635) UID: 1000 2026-02-19T21:11:26.632 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faac2335640 -1 rbd::mirror::InstanceWatcher: 0x564960eba380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13491 object: (2) No such file or directory 2026-02-19T21:11:26.632 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faac2335640 -1 rbd::mirror::InstanceWatcher: 0x564960ebaa80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13491 object: (2) No such file or directory 2026-02-19T21:11:26.632 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faabbb28640 -1 rbd::mirror::InstanceWatcher: 0x564960c24000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.632 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faabc329640 -1 rbd::mirror::InstanceWatcher: 0x564961263880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.632 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faabc329640 -1 rbd::mirror::InstanceWatcher: 0x564960eba700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.632 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faab5b1c640 -1 rbd::mirror::MirrorStatusUpdater 0x5649609e2fc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.632 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faac2b36640 -1 rbd::mirror::NamespaceReplayer: 0x56495f6c8a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.632 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faab531b640 -1 rbd::mirror::MirrorStatusUpdater 0x5649609e30e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faab531b640 -1 rbd::mirror::MirrorStatusUpdater 0x5649609e2ea0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faac2b36640 -1 rbd::mirror::NamespaceReplayer: 0x56495f6c8e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faac2b36640 -1 rbd::mirror::NamespaceReplayer: 0x56495f6c9180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faabbb28640 -1 rbd::mirror::MirrorStatusUpdater 0x564960a40240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faac2b36640 -1 rbd::mirror::NamespaceReplayer: 0x56495f6c9180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faaae30d640 -1 rbd::mirror::InstanceWatcher: 0x564960eba380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faaae30d640 -1 rbd::mirror::InstanceWatcher: 0x564960ebaa80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faaaeb0e640 -1 rbd::mirror::MirrorStatusUpdater 0x564960cccd80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faaaeb0e640 -1 rbd::mirror::MirrorStatusUpdater 0x564960cccea0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faac2b36640 -1 rbd::mirror::NamespaceReplayer: 0x564960b3c700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faac2335640 -1 rbd::mirror::NamespaceReplayer: 0x564960b3cc40 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.623+0000 7faac2b36640 -1 rbd::mirror::LeaderWatcher: 0x564960f3c900 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.646 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Stopped 2026-02-19T21:11:26.646 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-19T21:11:26.661 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.0:waiting for process to exit 2026-02-19T21:11:26.661 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:11:26.698 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7be479640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 42542) UID: 1000 2026-02-19T21:11:26.698 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b746b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x564fed39e900 handle_unregister_watcher: error unregistering watcher for trash directory: (107) Transport endpoint is not connected 2026-02-19T21:11:26.698 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b746b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x564fed39e6c0 handle_unregister_watcher: error unregistering watcher for trash directory: (107) Transport endpoint is not connected 2026-02-19T21:11:26.698 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b746b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x564fed39fb00 handle_unregister_watcher: error unregistering watcher for trash directory: (107) Transport endpoint is not connected 2026-02-19T21:11:26.698 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b746b640 -1 rbd::mirror::PoolWatcher: 0x564fed3f6000 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.698 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b045d640 -1 rbd::mirror::PoolWatcher: 0x564fed3f6780 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.698 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b746b640 -1 rbd::mirror::PoolWatcher: 0x564fecd410e0 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.698 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b045d640 -1 rbd::mirror::PoolWatcher: 0x564fecd40960 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.698 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b746b640 -1 rbd::mirror::PoolWatcher: 0x564fecd414a0 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b045d640 -1 rbd::mirror::PoolWatcher: 0x564fed3f61e0 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a9c50640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x564fed39f8c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a9c50640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x564fed39e000 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a944f640 -1 rbd::mirror::PoolWatcher: 0x564fed2901e0 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a9c50640 -1 rbd::mirror::PoolWatcher: 0x564fed380000 operator(): error unregistering watcher for rbd_mirroring object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bd477640 -1 rbd::mirror::InstanceWatcher: 0x564fec64a000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6385 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bd477640 -1 rbd::mirror::InstanceWatcher: 0x564fec64a700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6385 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bd477640 -1 rbd::mirror::InstanceWatcher: 0x564fec64a380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6385 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a3443640 -1 rbd::mirror::PoolWatcher: 0x564fed2905a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a3443640 -1 rbd::mirror::PoolWatcher: 0x564fed3801e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b6c6a640 -1 rbd::mirror::InstanceWatcher: 0x564fec64a000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.699 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b746b640 -1 rbd::mirror::InstanceWatcher: 0x564fec64a380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b746b640 -1 rbd::mirror::InstanceWatcher: 0x564fec64a700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bdc78640 -1 rbd::mirror::InstanceWatcher: 0x564fec64aa80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6398 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bdc78640 -1 rbd::mirror::InstanceWatcher: 0x564fec9a4380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6398 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b0c5e640 -1 rbd::mirror::MirrorStatusUpdater 0x564fec504360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b0c5e640 -1 rbd::mirror::MirrorStatusUpdater 0x564fec504480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bdc78640 -1 rbd::mirror::NamespaceReplayer: 0x564feb3a6e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b0c5e640 -1 rbd::mirror::MirrorStatusUpdater 0x564fec5045a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bdc78640 -1 rbd::mirror::NamespaceReplayer: 0x564feb3a6000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bdc78640 -1 rbd::mirror::NamespaceReplayer: 0x564feb3a7180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b6c6a640 -1 rbd::mirror::MirrorStatusUpdater 0x564fec504120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b6c6a640 -1 rbd::mirror::MirrorStatusUpdater 0x564fec504000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7b6c6a640 -1 rbd::mirror::MirrorStatusUpdater 0x564fec504240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bd477640 -1 rbd::mirror::NamespaceReplayer: 0x564feb3a6e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bd477640 -1 rbd::mirror::NamespaceReplayer: 0x564feb3a6000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bd477640 -1 rbd::mirror::NamespaceReplayer: 0x564feb3a7180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a944f640 -1 rbd::mirror::InstanceWatcher: 0x564fec64aa80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a944f640 -1 rbd::mirror::InstanceWatcher: 0x564fec9a4380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a2c42640 -1 rbd::mirror::MirrorStatusUpdater 0x564fec504b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a2c42640 -1 rbd::mirror::MirrorStatusUpdater 0x564fec504900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bdc78640 -1 rbd::mirror::NamespaceReplayer: 0x564fec634a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bdc78640 -1 rbd::mirror::NamespaceReplayer: 0x564fec634700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a9c50640 -1 rbd::mirror::MirrorStatusUpdater 0x564fec9bc900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7a9c50640 -1 rbd::mirror::MirrorStatusUpdater 0x564fec9bca20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.702 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bd477640 -1 rbd::mirror::NamespaceReplayer: 0x564fec634a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.702 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bd477640 -1 rbd::mirror::NamespaceReplayer: 0x564fec634700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.702 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bdc78640 -1 rbd::mirror::LeaderWatcher: 0x564feca48900 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-19T21:11:26.702 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bdc78640 -1 rbd::mirror::LeaderWatcher: 0x564feca48900 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-19T21:11:26.702 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.687+0000 7ff7bd477640 -1 rbd::mirror::LeaderWatcher: 0x564feca48900 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.702 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.691+0000 7ff7bd477640 -1 rbd::mirror::LeaderWatcher: 0x564fec426000 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-19T21:11:26.702 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.691+0000 7ff7bd477640 -1 rbd::mirror::LeaderWatcher: 0x564fec426000 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-19T21:11:26.702 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.691+0000 7ff7bdc78640 -1 rbd::mirror::LeaderWatcher: 0x564fec426000 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.713 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Stopped 2026-02-19T21:11:26.713 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-19T21:11:26.732 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.3:waiting for process to exit 2026-02-19T21:11:26.732 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:11:26.762 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaac26a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 79937) UID: 1000 2026-02-19T21:11:26.762 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaaba69640 -1 rbd::mirror::InstanceWatcher: 0x55fa56997880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4604 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.762 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaaba69640 -1 rbd::mirror::InstanceWatcher: 0x55fa57b47c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4604 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.762 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaaba69640 -1 rbd::mirror::InstanceWatcher: 0x55fa57b46000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4604 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.762 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaaba69640 -1 rbd::mirror::InstanceWatcher: 0x55fa57c86e00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4608 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.762 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaab268640 -1 rbd::mirror::InstanceWatcher: 0x55fa57c86700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4608 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.763 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaa525c640 -1 rbd::mirror::InstanceWatcher: 0x55fa56997880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.763 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faa9e24e640 -1 rbd::mirror::MirrorStatusUpdater 0x55fa579e4fc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.763 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaaba69640 -1 rbd::mirror::NamespaceReplayer: 0x55fa56904380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.763 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaa4a5b640 -1 rbd::mirror::InstanceWatcher: 0x55fa57b47c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.763 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaa4a5b640 -1 rbd::mirror::MirrorStatusUpdater 0x55fa57a3a000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.763 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaa4a5b640 -1 rbd::mirror::InstanceWatcher: 0x55fa57b46000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.763 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaab268640 -1 rbd::mirror::NamespaceReplayer: 0x55fa56904380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.763 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faa9ea4f640 -1 rbd::mirror::MirrorStatusUpdater 0x55fa579e50e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.764 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faa97240640 -1 rbd::mirror::InstanceWatcher: 0x55fa57c86e00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.764 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faa9ea4f640 -1 rbd::mirror::MirrorStatusUpdater 0x55fa579e4ea0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.764 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaab268640 -1 rbd::mirror::NamespaceReplayer: 0x55fa56904e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.764 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaab268640 -1 rbd::mirror::NamespaceReplayer: 0x55fa56904000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.764 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaa525c640 -1 rbd::mirror::MirrorStatusUpdater 0x55fa57a3a240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.764 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaaba69640 -1 rbd::mirror::NamespaceReplayer: 0x55fa56904e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.764 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaa4a5b640 -1 rbd::mirror::MirrorStatusUpdater 0x55fa57a3a120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.764 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaaba69640 -1 rbd::mirror::NamespaceReplayer: 0x55fa56904000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.765 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faa90a33640 -1 rbd::mirror::MirrorStatusUpdater 0x55fa57cb7320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.765 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaab268640 -1 rbd::mirror::NamespaceReplayer: 0x55fa57c46e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.765 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faa97a41640 -1 rbd::mirror::MirrorStatusUpdater 0x55fa57cb6fc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.765 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaaba69640 -1 rbd::mirror::NamespaceReplayer: 0x55fa57c46e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.765 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faa97240640 -1 rbd::mirror::InstanceWatcher: 0x55fa57c86700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.765 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faa91234640 -1 rbd::mirror::MirrorStatusUpdater 0x55fa57cb7680 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.765 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaab268640 -1 rbd::mirror::NamespaceReplayer: 0x55fa57c46fc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.765 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faa97a41640 -1 rbd::mirror::MirrorStatusUpdater 0x55fa57cb70e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.766 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaaba69640 -1 rbd::mirror::NamespaceReplayer: 0x55fa57c46fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.766 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaaba69640 -1 rbd::mirror::LeaderWatcher: 0x55fa57f64900 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.766 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial151.stderr:2026-02-19T21:11:26.751+0000 7faaab268640 -1 rbd::mirror::LeaderWatcher: 0x55fa57972000 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.780 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Stopped 2026-02-19T21:11:26.780 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-19T21:11:26.799 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.2:waiting for process to exit 2026-02-19T21:11:26.799 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:11:26.825 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4851d2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 106091) UID: 1000 2026-02-19T21:11:26.828 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::InstanceWatcher: 0x55cfcf3ea000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4900 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::InstanceWatcher: 0x55cfcf3ea700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4900 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::InstanceWatcher: 0x55cfcf3ea380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4900 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::InstanceWatcher: 0x55cfcf3eaa80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4903 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::InstanceWatcher: 0x55cfcf4e2700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4903 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb47d9c3640 -1 rbd::mirror::InstanceWatcher: 0x55cfcf3ea000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4779b7640 -1 rbd::mirror::MirrorStatusUpdater 0x55cfcf282360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb47e1c4640 -1 rbd::mirror::InstanceWatcher: 0x55cfcf3ea380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb47e1c4640 -1 rbd::mirror::InstanceWatcher: 0x55cfcf3ea700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::NamespaceReplayer: 0x55cfcf280000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4709a9640 -1 rbd::mirror::InstanceWatcher: 0x55cfcf3eaa80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4709a9640 -1 rbd::mirror::InstanceWatcher: 0x55cfcf4e2700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb47d9c3640 -1 rbd::mirror::MirrorStatusUpdater 0x55cfcf282000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::NamespaceReplayer: 0x55cfcf280000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4771b6640 -1 rbd::mirror::MirrorStatusUpdater 0x55cfcf282480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4771b6640 -1 rbd::mirror::MirrorStatusUpdater 0x55cfcf2825a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::NamespaceReplayer: 0x55cfcf280380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::NamespaceReplayer: 0x55cfcf2801c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb47e1c4640 -1 rbd::mirror::MirrorStatusUpdater 0x55cfcf282240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb47e1c4640 -1 rbd::mirror::MirrorStatusUpdater 0x55cfcf282120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4849d1640 -1 rbd::mirror::NamespaceReplayer: 0x55cfcf280380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4849d1640 -1 rbd::mirror::NamespaceReplayer: 0x55cfcf2801c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.830 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb46999b640 -1 rbd::mirror::MirrorStatusUpdater 0x55cfcf51f680 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-19T21:11:26.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb46999b640 -1 rbd::mirror::MirrorStatusUpdater 0x55cfcf51f320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-19T21:11:26.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4849d1640 -1 rbd::mirror::NamespaceReplayer: 0x55cfcf280fc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-19T21:11:26.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4849d1640 -1 rbd::mirror::NamespaceReplayer: 0x55cfcf281180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-19T21:11:26.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4701a8640 -1 rbd::mirror::MirrorStatusUpdater 0x55cfcf51efc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4701a8640 -1 rbd::mirror::MirrorStatusUpdater 0x55cfcf51f0e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::NamespaceReplayer: 0x55cfcf280fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::NamespaceReplayer: 0x55cfcf281180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.831 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4841d0640 -1 rbd::mirror::LeaderWatcher: 0x55cfcf71c900 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.832 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial151.stderr:2026-02-19T21:11:26.819+0000 7fb4849d1640 -1 rbd::mirror::LeaderWatcher: 0x55cfcf1c8000 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.845 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Stopped 2026-02-19T21:11:26.845 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-19T21:11:26.864 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.1:waiting for process to exit 2026-02-19T21:11:26.864 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:11:26.889 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24ea6e9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 125807) UID: 1000 2026-02-19T21:11:26.891 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e96e7640 -1 rbd::mirror::InstanceWatcher: 0x55d7cfc1c700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5406 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e96e7640 -1 rbd::mirror::InstanceWatcher: 0x55d7cfc1c380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5406 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e96e7640 -1 rbd::mirror::InstanceWatcher: 0x55d7cfc1c000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5406 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24d56bf640 -1 rbd::mirror::InstanceWatcher: 0x55d7cff7a380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24d5ec0640 -1 rbd::mirror::InstanceWatcher: 0x55d7cfc1ca80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e2eda640 -1 rbd::mirror::InstanceWatcher: 0x55d7cfc1c700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e36db640 -1 rbd::mirror::InstanceWatcher: 0x55d7cfc1c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e36db640 -1 rbd::mirror::InstanceWatcher: 0x55d7cfc1c380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e36db640 -1 rbd::mirror::MirrorStatusUpdater 0x55d7cfad6000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e96e7640 -1 rbd::mirror::NamespaceReplayer: 0x55d7ce787180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24cf6b3640 -1 rbd::mirror::MirrorStatusUpdater 0x55d7cfd49680 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e9ee8640 -1 rbd::mirror::NamespaceReplayer: 0x55d7cfad8fc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-19T21:11:26.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e2eda640 -1 rbd::mirror::MirrorStatusUpdater 0x55d7cfad6240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.893 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24ceeb2640 -1 rbd::mirror::MirrorStatusUpdater 0x55d7cfd49320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-19T21:11:26.893 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e2eda640 -1 rbd::mirror::MirrorStatusUpdater 0x55d7cfad6120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.893 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e9ee8640 -1 rbd::mirror::NamespaceReplayer: 0x55d7cfad81c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.893 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e9ee8640 -1 rbd::mirror::NamespaceReplayer: 0x55d7cfad9180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-19T21:11:26.893 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e9ee8640 -1 rbd::mirror::NamespaceReplayer: 0x55d7cfad8000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.893 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24d56bf640 -1 rbd::mirror::MirrorStatusUpdater 0x55d7cfd490e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.893 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e9ee8640 -1 rbd::mirror::NamespaceReplayer: 0x55d7cfad9180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.893 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial151.stderr:2026-02-19T21:11:26.883+0000 7f24e9ee8640 -1 rbd::mirror::LeaderWatcher: 0x55d7cfa00000 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Stopped 2026-02-19T21:11:26.905 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-19T21:11:26.917 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.0:waiting for process to exit 2026-02-19T21:11:26.917 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:11:26.949 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0b81c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 128085) UID: 1000 2026-02-19T21:11:26.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0a81a640 -1 rbd::mirror::InstanceWatcher: 0x55cf5c7ce380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5506 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0a81a640 -1 rbd::mirror::InstanceWatcher: 0x55cf5c7ce700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5506 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0a81a640 -1 rbd::mirror::InstanceWatcher: 0x55cf5c7ce000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5506 object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbbf67f2640 -1 rbd::mirror::InstanceWatcher: 0x55cf5c7cea80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0480e640 -1 rbd::mirror::InstanceWatcher: 0x55cf5c7ce380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbbf6ff3640 -1 rbd::mirror::InstanceWatcher: 0x55cf5cba0380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0400d640 -1 rbd::mirror::InstanceWatcher: 0x55cf5c7ce700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0400d640 -1 rbd::mirror::InstanceWatcher: 0x55cf5c7ce000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0400d640 -1 rbd::mirror::MirrorStatusUpdater 0x55cf5c70c000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0b01b640 -1 rbd::mirror::NamespaceReplayer: 0x55cf5b3ad500 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0480e640 -1 rbd::mirror::MirrorStatusUpdater 0x55cf5c70c120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0a81a640 -1 rbd::mirror::NamespaceReplayer: 0x55cf5b3ad340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0400d640 -1 rbd::mirror::MirrorStatusUpdater 0x55cf5c70c240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (107) Transport endpoint is not connected 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0a81a640 -1 rbd::mirror::NamespaceReplayer: 0x55cf5b3aca80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (107) Transport endpoint is not connected 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbbf07e6640 -1 rbd::mirror::MirrorStatusUpdater 0x55cf5c9637a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbbf07e6640 -1 rbd::mirror::MirrorStatusUpdater 0x55cf5c963440 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0b01b640 -1 rbd::mirror::NamespaceReplayer: 0x55cf5c81c8c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-19T21:11:26.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0b01b640 -1 rbd::mirror::NamespaceReplayer: 0x55cf5c81ca80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-19T21:11:26.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial151.stderr:2026-02-19T21:11:26.943+0000 7fbc0b01b640 -1 rbd::mirror::LeaderWatcher: 0x55cf5c64a000 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (107) Transport endpoint is not connected 2026-02-19T21:11:26.966 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Stopped 2026-02-19T21:11:26.966 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-19T21:11:26.977 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-19T21:11:26.977 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-19T21:11:27.164 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T21:11:27.165 INFO:teuthology.orchestra.run.trial151.stderr:dumped all 2026-02-19T21:11:27.174 INFO:teuthology.orchestra.run.trial151.stdout:{"pg_ready":true,"pg_map":{"version":1686,"stamp":"2026-02-19T21:11:26.902320+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":227,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4735540,"kb_used_data":15684,"kb_used_omap":1179,"kb_used_meta":4718628,"kb_avail":278379980,"statfs":{"total":289910292480,"available":285061099520,"internally_reserved":0,"allocated":16060416,"data_stored":8073950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1207463,"internal_metadata":4831875929},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-896455906,"num_objects":-198,"num_object_clones":0,"num_object_copies":-396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-198,"num_whiteouts":0,"num_read":-2369771,"num_read_kb":-90003014,"num_write":-2291455,"num_write_kb":-89006296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000920"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":844,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.841290+0000","last_change":"2026-02-19T20:29:11.745770+0000","last_active":"2026-02-19T21:11:24.841290+0000","last_peered":"2026-02-19T21:11:24.841290+0000","last_clean":"2026-02-19T21:11:24.841290+0000","last_became_active":"2026-02-19T20:29:09.741410+0000","last_became_peered":"2026-02-19T20:29:09.741410+0000","last_unstale":"2026-02-19T21:11:24.841290+0000","last_undegraded":"2026-02-19T21:11:24.841290+0000","last_fullsized":"2026-02-19T21:11:24.841290+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:27:41.302717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052313100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":844,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.842400+0000","last_change":"2026-02-19T20:29:11.745905+0000","last_active":"2026-02-19T21:11:24.842400+0000","last_peered":"2026-02-19T21:11:24.842400+0000","last_clean":"2026-02-19T21:11:24.842400+0000","last_became_active":"2026-02-19T20:29:09.741207+0000","last_became_peered":"2026-02-19T20:29:09.741207+0000","last_unstale":"2026-02-19T21:11:24.842400+0000","last_undegraded":"2026-02-19T21:11:24.842400+0000","last_fullsized":"2026-02-19T21:11:24.842400+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:30:59.713739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066944300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":845,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.844066+0000","last_change":"2026-02-19T20:29:11.745724+0000","last_active":"2026-02-19T21:11:24.844066+0000","last_peered":"2026-02-19T21:11:24.844066+0000","last_clean":"2026-02-19T21:11:24.844066+0000","last_became_active":"2026-02-19T20:29:09.741235+0000","last_became_peered":"2026-02-19T20:29:09.741235+0000","last_unstale":"2026-02-19T21:11:24.844066+0000","last_undegraded":"2026-02-19T21:11:24.844066+0000","last_fullsized":"2026-02-19T21:11:24.844066+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:34:53.768773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051114900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":845,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.840274+0000","last_change":"2026-02-19T20:29:11.745539+0000","last_active":"2026-02-19T21:11:24.840274+0000","last_peered":"2026-02-19T21:11:24.840274+0000","last_clean":"2026-02-19T21:11:24.840274+0000","last_became_active":"2026-02-19T20:29:09.740922+0000","last_became_peered":"2026-02-19T20:29:09.740922+0000","last_unstale":"2026-02-19T21:11:24.840274+0000","last_undegraded":"2026-02-19T21:11:24.840274+0000","last_fullsized":"2026-02-19T21:11:24.840274+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:44:16.698249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353877,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":848,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.494714+0000","last_change":"2026-02-19T20:29:11.743941+0000","last_active":"2026-02-19T21:11:24.494714+0000","last_peered":"2026-02-19T21:11:24.494714+0000","last_clean":"2026-02-19T21:11:24.494714+0000","last_became_active":"2026-02-19T20:29:09.740605+0000","last_became_peered":"2026-02-19T20:29:09.740605+0000","last_unstale":"2026-02-19T21:11:24.494714+0000","last_undegraded":"2026-02-19T21:11:24.494714+0000","last_fullsized":"2026-02-19T21:11:24.494714+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T00:12:21.879309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083745499999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":848,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:25.967031+0000","last_change":"2026-02-19T20:29:11.741215+0000","last_active":"2026-02-19T21:11:25.967031+0000","last_peered":"2026-02-19T21:11:25.967031+0000","last_clean":"2026-02-19T21:11:25.967031+0000","last_became_active":"2026-02-19T20:29:09.740655+0000","last_became_peered":"2026-02-19T20:29:09.740655+0000","last_unstale":"2026-02-19T21:11:25.967031+0000","last_undegraded":"2026-02-19T21:11:25.967031+0000","last_fullsized":"2026-02-19T21:11:25.967031+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:52:18.535895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037179999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":849,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:25.965916+0000","last_change":"2026-02-19T20:29:11.741323+0000","last_active":"2026-02-19T21:11:25.965916+0000","last_peered":"2026-02-19T21:11:25.965916+0000","last_clean":"2026-02-19T21:11:25.965916+0000","last_became_active":"2026-02-19T20:29:09.740330+0000","last_became_peered":"2026-02-19T20:29:09.740330+0000","last_unstale":"2026-02-19T21:11:25.965916+0000","last_undegraded":"2026-02-19T21:11:25.965916+0000","last_fullsized":"2026-02-19T21:11:25.965916+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:54:37.429972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041288700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":846,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.843514+0000","last_change":"2026-02-19T20:29:11.745555+0000","last_active":"2026-02-19T21:11:24.843514+0000","last_peered":"2026-02-19T21:11:24.843514+0000","last_clean":"2026-02-19T21:11:24.843514+0000","last_became_active":"2026-02-19T20:29:09.740070+0000","last_became_peered":"2026-02-19T20:29:09.740070+0000","last_unstale":"2026-02-19T21:11:24.843514+0000","last_undegraded":"2026-02-19T21:11:24.843514+0000","last_fullsized":"2026-02-19T21:11:24.843514+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T22:03:17.974819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036770299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":935,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.842526+0000","last_change":"2026-02-19T20:29:07.943283+0000","last_active":"2026-02-19T21:11:24.842526+0000","last_peered":"2026-02-19T21:11:24.842526+0000","last_clean":"2026-02-19T21:11:24.842526+0000","last_became_active":"2026-02-19T20:29:07.942262+0000","last_became_peered":"2026-02-19T20:29:07.942262+0000","last_unstale":"2026-02-19T21:11:24.842526+0000","last_undegraded":"2026-02-19T21:11:24.842526+0000","last_fullsized":"2026-02-19T21:11:24.842526+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_clean_scrub_stamp":"2026-02-19T20:29:06.726163+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:37:51.447976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738880,"num_pgs":79,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1343872,"kb_used_data":4848,"kb_used_omap":344,"kb_used_meta":1338663,"kb_avail":93027968,"statfs":{"total":96636764160,"available":95260639232,"internally_reserved":0,"allocated":4964352,"data_stored":2302147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":352789,"internal_metadata":1370791403},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738880,"num_pgs":99,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1676604,"kb_used_data":5420,"kb_used_omap":494,"kb_used_meta":1670673,"kb_avail":92695236,"statfs":{"total":96636764160,"available":94919921664,"internally_reserved":0,"allocated":5550080,"data_stored":2885911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":506419,"internal_metadata":1710769613},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738879,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1715064,"kb_used_data":5416,"kb_used_omap":340,"kb_used_meta":1709291,"kb_avail":92656776,"statfs":{"total":96636764160,"available":94880538624,"internally_reserved":0,"allocated":5545984,"data_stored":2885892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":348255,"internal_metadata":1750314913},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T21:11:27.176 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-19T21:11:27.329 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T21:11:27.330 INFO:teuthology.orchestra.run.trial151.stderr:dumped all 2026-02-19T21:11:27.340 INFO:teuthology.orchestra.run.trial151.stdout:{"pg_ready":true,"pg_map":{"version":1686,"stamp":"2026-02-19T21:11:26.902320+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":227,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4735540,"kb_used_data":15684,"kb_used_omap":1179,"kb_used_meta":4718628,"kb_avail":278379980,"statfs":{"total":289910292480,"available":285061099520,"internally_reserved":0,"allocated":16060416,"data_stored":8073950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1207463,"internal_metadata":4831875929},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-896455906,"num_objects":-198,"num_object_clones":0,"num_object_copies":-396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-198,"num_whiteouts":0,"num_read":-2369771,"num_read_kb":-90003014,"num_write":-2291455,"num_write_kb":-89006296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000920"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":844,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.841290+0000","last_change":"2026-02-19T20:29:11.745770+0000","last_active":"2026-02-19T21:11:24.841290+0000","last_peered":"2026-02-19T21:11:24.841290+0000","last_clean":"2026-02-19T21:11:24.841290+0000","last_became_active":"2026-02-19T20:29:09.741410+0000","last_became_peered":"2026-02-19T20:29:09.741410+0000","last_unstale":"2026-02-19T21:11:24.841290+0000","last_undegraded":"2026-02-19T21:11:24.841290+0000","last_fullsized":"2026-02-19T21:11:24.841290+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:27:41.302717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052313100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":844,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.842400+0000","last_change":"2026-02-19T20:29:11.745905+0000","last_active":"2026-02-19T21:11:24.842400+0000","last_peered":"2026-02-19T21:11:24.842400+0000","last_clean":"2026-02-19T21:11:24.842400+0000","last_became_active":"2026-02-19T20:29:09.741207+0000","last_became_peered":"2026-02-19T20:29:09.741207+0000","last_unstale":"2026-02-19T21:11:24.842400+0000","last_undegraded":"2026-02-19T21:11:24.842400+0000","last_fullsized":"2026-02-19T21:11:24.842400+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:30:59.713739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066944300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":845,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.844066+0000","last_change":"2026-02-19T20:29:11.745724+0000","last_active":"2026-02-19T21:11:24.844066+0000","last_peered":"2026-02-19T21:11:24.844066+0000","last_clean":"2026-02-19T21:11:24.844066+0000","last_became_active":"2026-02-19T20:29:09.741235+0000","last_became_peered":"2026-02-19T20:29:09.741235+0000","last_unstale":"2026-02-19T21:11:24.844066+0000","last_undegraded":"2026-02-19T21:11:24.844066+0000","last_fullsized":"2026-02-19T21:11:24.844066+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:34:53.768773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051114900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":845,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.840274+0000","last_change":"2026-02-19T20:29:11.745539+0000","last_active":"2026-02-19T21:11:24.840274+0000","last_peered":"2026-02-19T21:11:24.840274+0000","last_clean":"2026-02-19T21:11:24.840274+0000","last_became_active":"2026-02-19T20:29:09.740922+0000","last_became_peered":"2026-02-19T20:29:09.740922+0000","last_unstale":"2026-02-19T21:11:24.840274+0000","last_undegraded":"2026-02-19T21:11:24.840274+0000","last_fullsized":"2026-02-19T21:11:24.840274+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:44:16.698249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353877,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":848,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.494714+0000","last_change":"2026-02-19T20:29:11.743941+0000","last_active":"2026-02-19T21:11:24.494714+0000","last_peered":"2026-02-19T21:11:24.494714+0000","last_clean":"2026-02-19T21:11:24.494714+0000","last_became_active":"2026-02-19T20:29:09.740605+0000","last_became_peered":"2026-02-19T20:29:09.740605+0000","last_unstale":"2026-02-19T21:11:24.494714+0000","last_undegraded":"2026-02-19T21:11:24.494714+0000","last_fullsized":"2026-02-19T21:11:24.494714+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T00:12:21.879309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083745499999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":848,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:25.967031+0000","last_change":"2026-02-19T20:29:11.741215+0000","last_active":"2026-02-19T21:11:25.967031+0000","last_peered":"2026-02-19T21:11:25.967031+0000","last_clean":"2026-02-19T21:11:25.967031+0000","last_became_active":"2026-02-19T20:29:09.740655+0000","last_became_peered":"2026-02-19T20:29:09.740655+0000","last_unstale":"2026-02-19T21:11:25.967031+0000","last_undegraded":"2026-02-19T21:11:25.967031+0000","last_fullsized":"2026-02-19T21:11:25.967031+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:52:18.535895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037179999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":849,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:25.965916+0000","last_change":"2026-02-19T20:29:11.741323+0000","last_active":"2026-02-19T21:11:25.965916+0000","last_peered":"2026-02-19T21:11:25.965916+0000","last_clean":"2026-02-19T21:11:25.965916+0000","last_became_active":"2026-02-19T20:29:09.740330+0000","last_became_peered":"2026-02-19T20:29:09.740330+0000","last_unstale":"2026-02-19T21:11:25.965916+0000","last_undegraded":"2026-02-19T21:11:25.965916+0000","last_fullsized":"2026-02-19T21:11:25.965916+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:54:37.429972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041288700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":846,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.843514+0000","last_change":"2026-02-19T20:29:11.745555+0000","last_active":"2026-02-19T21:11:24.843514+0000","last_peered":"2026-02-19T21:11:24.843514+0000","last_clean":"2026-02-19T21:11:24.843514+0000","last_became_active":"2026-02-19T20:29:09.740070+0000","last_became_peered":"2026-02-19T20:29:09.740070+0000","last_unstale":"2026-02-19T21:11:24.843514+0000","last_undegraded":"2026-02-19T21:11:24.843514+0000","last_fullsized":"2026-02-19T21:11:24.843514+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T22:03:17.974819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036770299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":935,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.842526+0000","last_change":"2026-02-19T20:29:07.943283+0000","last_active":"2026-02-19T21:11:24.842526+0000","last_peered":"2026-02-19T21:11:24.842526+0000","last_clean":"2026-02-19T21:11:24.842526+0000","last_became_active":"2026-02-19T20:29:07.942262+0000","last_became_peered":"2026-02-19T20:29:07.942262+0000","last_unstale":"2026-02-19T21:11:24.842526+0000","last_undegraded":"2026-02-19T21:11:24.842526+0000","last_fullsized":"2026-02-19T21:11:24.842526+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_clean_scrub_stamp":"2026-02-19T20:29:06.726163+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:37:51.447976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738880,"num_pgs":79,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1343872,"kb_used_data":4848,"kb_used_omap":344,"kb_used_meta":1338663,"kb_avail":93027968,"statfs":{"total":96636764160,"available":95260639232,"internally_reserved":0,"allocated":4964352,"data_stored":2302147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":352789,"internal_metadata":1370791403},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738880,"num_pgs":99,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1676604,"kb_used_data":5420,"kb_used_omap":494,"kb_used_meta":1670673,"kb_avail":92695236,"statfs":{"total":96636764160,"available":94919921664,"internally_reserved":0,"allocated":5550080,"data_stored":2885911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":506419,"internal_metadata":1710769613},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738879,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1715064,"kb_used_data":5416,"kb_used_omap":340,"kb_used_meta":1709291,"kb_avail":92656776,"statfs":{"total":96636764160,"available":94880538624,"internally_reserved":0,"allocated":5545984,"data_stored":2885892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":348255,"internal_metadata":1750314913},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T21:11:27.342 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-19T21:11:27.342 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-19T21:11:27.494 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T21:11:27.495 INFO:teuthology.orchestra.run.trial151.stderr:dumped all 2026-02-19T21:11:27.505 INFO:teuthology.orchestra.run.trial151.stdout:{"pg_ready":true,"pg_map":{"version":1686,"stamp":"2026-02-19T21:11:26.902320+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":227,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4735540,"kb_used_data":15684,"kb_used_omap":1179,"kb_used_meta":4718628,"kb_avail":278379980,"statfs":{"total":289910292480,"available":285061099520,"internally_reserved":0,"allocated":16060416,"data_stored":8073950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1207463,"internal_metadata":4831875929},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-896455906,"num_objects":-198,"num_object_clones":0,"num_object_copies":-396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-198,"num_whiteouts":0,"num_read":-2369771,"num_read_kb":-90003014,"num_write":-2291455,"num_write_kb":-89006296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000920"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":844,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.841290+0000","last_change":"2026-02-19T20:29:11.745770+0000","last_active":"2026-02-19T21:11:24.841290+0000","last_peered":"2026-02-19T21:11:24.841290+0000","last_clean":"2026-02-19T21:11:24.841290+0000","last_became_active":"2026-02-19T20:29:09.741410+0000","last_became_peered":"2026-02-19T20:29:09.741410+0000","last_unstale":"2026-02-19T21:11:24.841290+0000","last_undegraded":"2026-02-19T21:11:24.841290+0000","last_fullsized":"2026-02-19T21:11:24.841290+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:27:41.302717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052313100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":844,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.842400+0000","last_change":"2026-02-19T20:29:11.745905+0000","last_active":"2026-02-19T21:11:24.842400+0000","last_peered":"2026-02-19T21:11:24.842400+0000","last_clean":"2026-02-19T21:11:24.842400+0000","last_became_active":"2026-02-19T20:29:09.741207+0000","last_became_peered":"2026-02-19T20:29:09.741207+0000","last_unstale":"2026-02-19T21:11:24.842400+0000","last_undegraded":"2026-02-19T21:11:24.842400+0000","last_fullsized":"2026-02-19T21:11:24.842400+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:30:59.713739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066944300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":845,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.844066+0000","last_change":"2026-02-19T20:29:11.745724+0000","last_active":"2026-02-19T21:11:24.844066+0000","last_peered":"2026-02-19T21:11:24.844066+0000","last_clean":"2026-02-19T21:11:24.844066+0000","last_became_active":"2026-02-19T20:29:09.741235+0000","last_became_peered":"2026-02-19T20:29:09.741235+0000","last_unstale":"2026-02-19T21:11:24.844066+0000","last_undegraded":"2026-02-19T21:11:24.844066+0000","last_fullsized":"2026-02-19T21:11:24.844066+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:34:53.768773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051114900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":845,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.840274+0000","last_change":"2026-02-19T20:29:11.745539+0000","last_active":"2026-02-19T21:11:24.840274+0000","last_peered":"2026-02-19T21:11:24.840274+0000","last_clean":"2026-02-19T21:11:24.840274+0000","last_became_active":"2026-02-19T20:29:09.740922+0000","last_became_peered":"2026-02-19T20:29:09.740922+0000","last_unstale":"2026-02-19T21:11:24.840274+0000","last_undegraded":"2026-02-19T21:11:24.840274+0000","last_fullsized":"2026-02-19T21:11:24.840274+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:44:16.698249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353877,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":848,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.494714+0000","last_change":"2026-02-19T20:29:11.743941+0000","last_active":"2026-02-19T21:11:24.494714+0000","last_peered":"2026-02-19T21:11:24.494714+0000","last_clean":"2026-02-19T21:11:24.494714+0000","last_became_active":"2026-02-19T20:29:09.740605+0000","last_became_peered":"2026-02-19T20:29:09.740605+0000","last_unstale":"2026-02-19T21:11:24.494714+0000","last_undegraded":"2026-02-19T21:11:24.494714+0000","last_fullsized":"2026-02-19T21:11:24.494714+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T00:12:21.879309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083745499999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":848,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:25.967031+0000","last_change":"2026-02-19T20:29:11.741215+0000","last_active":"2026-02-19T21:11:25.967031+0000","last_peered":"2026-02-19T21:11:25.967031+0000","last_clean":"2026-02-19T21:11:25.967031+0000","last_became_active":"2026-02-19T20:29:09.740655+0000","last_became_peered":"2026-02-19T20:29:09.740655+0000","last_unstale":"2026-02-19T21:11:25.967031+0000","last_undegraded":"2026-02-19T21:11:25.967031+0000","last_fullsized":"2026-02-19T21:11:25.967031+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:52:18.535895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037179999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":849,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:25.965916+0000","last_change":"2026-02-19T20:29:11.741323+0000","last_active":"2026-02-19T21:11:25.965916+0000","last_peered":"2026-02-19T21:11:25.965916+0000","last_clean":"2026-02-19T21:11:25.965916+0000","last_became_active":"2026-02-19T20:29:09.740330+0000","last_became_peered":"2026-02-19T20:29:09.740330+0000","last_unstale":"2026-02-19T21:11:25.965916+0000","last_undegraded":"2026-02-19T21:11:25.965916+0000","last_fullsized":"2026-02-19T21:11:25.965916+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:54:37.429972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041288700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":846,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.843514+0000","last_change":"2026-02-19T20:29:11.745555+0000","last_active":"2026-02-19T21:11:24.843514+0000","last_peered":"2026-02-19T21:11:24.843514+0000","last_clean":"2026-02-19T21:11:24.843514+0000","last_became_active":"2026-02-19T20:29:09.740070+0000","last_became_peered":"2026-02-19T20:29:09.740070+0000","last_unstale":"2026-02-19T21:11:24.843514+0000","last_undegraded":"2026-02-19T21:11:24.843514+0000","last_fullsized":"2026-02-19T21:11:24.843514+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T22:03:17.974819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036770299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":935,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.842526+0000","last_change":"2026-02-19T20:29:07.943283+0000","last_active":"2026-02-19T21:11:24.842526+0000","last_peered":"2026-02-19T21:11:24.842526+0000","last_clean":"2026-02-19T21:11:24.842526+0000","last_became_active":"2026-02-19T20:29:07.942262+0000","last_became_peered":"2026-02-19T20:29:07.942262+0000","last_unstale":"2026-02-19T21:11:24.842526+0000","last_undegraded":"2026-02-19T21:11:24.842526+0000","last_fullsized":"2026-02-19T21:11:24.842526+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_clean_scrub_stamp":"2026-02-19T20:29:06.726163+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:37:51.447976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738880,"num_pgs":79,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1343872,"kb_used_data":4848,"kb_used_omap":344,"kb_used_meta":1338663,"kb_avail":93027968,"statfs":{"total":96636764160,"available":95260639232,"internally_reserved":0,"allocated":4964352,"data_stored":2302147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":352789,"internal_metadata":1370791403},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738880,"num_pgs":99,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1676604,"kb_used_data":5420,"kb_used_omap":494,"kb_used_meta":1670673,"kb_avail":92695236,"statfs":{"total":96636764160,"available":94919921664,"internally_reserved":0,"allocated":5550080,"data_stored":2885911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":506419,"internal_metadata":1710769613},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738879,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1715064,"kb_used_data":5416,"kb_used_omap":340,"kb_used_meta":1709291,"kb_avail":92656776,"statfs":{"total":96636764160,"available":94880538624,"internally_reserved":0,"allocated":5545984,"data_stored":2885892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":348255,"internal_metadata":1750314913},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T21:11:27.507 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-19T21:11:27.664 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T21:11:27.664 INFO:teuthology.orchestra.run.trial151.stdout:{"epoch":418,"fsid":"a14e4c04-90a3-4ebc-956c-42208d4170cb","created":"2026-02-19T20:29:00.712510+0000","modified":"2026-02-19T21:11:25.948628+0000","last_up_change":"2026-02-19T20:29:05.723698+0000","last_in_change":"2026-02-19T20:29:01.351350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-19T20:29:06.483311+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-19T20:29:08.492120+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ddaf41b0-845c-4123-af03-f7dace7c6d41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6808","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6809","nonce":1469645752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6810","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6811","nonce":1469645752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6814","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6815","nonce":1469645752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6812","nonce":1469645752},{"type":"v1","addr":"10.20.193.151:6813","nonce":1469645752}]},"public_addr":"10.20.193.151:6809/1469645752","cluster_addr":"10.20.193.151:6811/1469645752","heartbeat_back_addr":"10.20.193.151:6815/1469645752","heartbeat_front_addr":"10.20.193.151:6813/1469645752","state":["exists","up"]},{"osd":1,"uuid":"31b161d3-699c-4d78-bf2f-2fe95de919b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6800","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6801","nonce":3847550616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6802","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6803","nonce":3847550616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6806","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6807","nonce":3847550616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6804","nonce":3847550616},{"type":"v1","addr":"10.20.193.151:6805","nonce":3847550616}]},"public_addr":"10.20.193.151:6801/3847550616","cluster_addr":"10.20.193.151:6803/3847550616","heartbeat_back_addr":"10.20.193.151:6807/3847550616","heartbeat_front_addr":"10.20.193.151:6805/3847550616","state":["exists","up"]},{"osd":2,"uuid":"e835eb22-9891-4406-8ec0-05ccd24a9828","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6816","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6817","nonce":3305594166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6818","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6819","nonce":3305594166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6822","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6823","nonce":3305594166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.151:6820","nonce":3305594166},{"type":"v1","addr":"10.20.193.151:6821","nonce":3305594166}]},"public_addr":"10.20.193.151:6817/3305594166","cluster_addr":"10.20.193.151:6819/3305594166","heartbeat_back_addr":"10.20.193.151:6823/3305594166","heartbeat_front_addr":"10.20.193.151:6821/3305594166","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-19T20:29:03.945678+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-19T20:29:04.021053+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-19T20:29:04.008479+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"10.20.193.151:0/1533655970":"2026-02-19T22:11:21.726842+0000","10.20.193.151:0/562023575":"2026-02-19T22:10:43.069937+0000","10.20.193.151:0/1676619138":"2026-02-19T22:09:16.722205+0000","10.20.193.151:0/135519560":"2026-02-19T22:07:24.460731+0000","10.20.193.151:0/1528192315":"2026-02-19T22:06:45.487662+0000","10.20.193.151:0/4234414887":"2026-02-19T22:05:22.410358+0000","10.20.193.151:0/2327182205":"2026-02-19T22:03:06.595048+0000","10.20.193.151:0/1810510207":"2026-02-19T21:56:29.006211+0000","10.20.193.151:0/1301126114":"2026-02-19T21:55:39.378596+0000","10.20.193.151:0/1245230349":"2026-02-19T22:08:39.782483+0000","10.20.193.151:0/977941667":"2026-02-19T21:55:23.505140+0000","10.20.193.151:0/3736095532":"2026-02-19T21:54:25.207210+0000","10.20.193.151:0/141196574":"2026-02-19T21:52:40.331588+0000","10.20.193.151:0/890184031":"2026-02-19T22:09:59.104640+0000","10.20.193.151:0/625110090":"2026-02-19T21:34:37.440235+0000","10.20.193.151:0/1054296874":"2026-02-19T21:57:09.103808+0000","10.20.193.151:0/1520845366":"2026-02-19T22:00:48.870640+0000","10.20.193.151:0/2460955521":"2026-02-19T21:53:55.513771+0000","10.20.193.151:0/1977229551":"2026-02-19T21:54:06.349297+0000","10.20.193.151:0/3612059768":"2026-02-19T22:06:07.439962+0000","10.20.193.151:0/2638394872":"2026-02-19T21:53:43.901993+0000","10.20.193.151:0/2313129795":"2026-02-19T22:00:17.654247+0000","10.20.193.151:0/3041662726":"2026-02-19T21:54:15.391439+0000","10.20.193.151:0/2340130320":"2026-02-19T21:52:53.772281+0000","10.20.193.151:0/3799991383":"2026-02-19T21:53:15.601212+0000","10.20.193.151:0/4232141039":"2026-02-19T21:33:15.362680+0000","10.20.193.151:0/238552999":"2026-02-19T21:53:05.429505+0000","10.20.193.151:0/1870186047":"2026-02-19T21:30:59.287064+0000","10.20.193.151:0/4269624687":"2026-02-19T21:42:00.555855+0000","10.20.193.151:0/891088824":"2026-02-19T21:57:49.174536+0000","10.20.193.151:0/1735411574":"2026-02-19T21:53:34.561494+0000","10.20.193.151:0/2819164190":"2026-02-19T21:38:16.473012+0000","10.20.193.151:0/2401726493":"2026-02-19T21:44:39.623501+0000","10.20.193.151:0/2607925177":"2026-02-19T22:01:43.534732+0000","10.20.193.151:0/3981333473":"2026-02-19T21:35:54.233573+0000","10.20.193.151:0/979454733":"2026-02-19T21:56:07.576989+0000","10.20.193.151:0/3745236004":"2026-02-19T21:59:19.035665+0000","10.20.193.151:0/3201331852":"2026-02-19T21:47:04.396269+0000","10.20.193.151:0/757236237":"2026-02-19T22:00:36.135738+0000","10.20.193.151:0/3986333120":"2026-02-19T21:37:09.348310+0000","10.20.193.151:0/1511865128":"2026-02-19T22:00:23.143173+0000","10.20.193.151:0/3497304825":"2026-02-19T22:01:23.851042+0000","10.20.193.151:0/3891271376":"2026-02-19T21:55:12.250811+0000","10.20.193.151:0/3547121887":"2026-02-19T22:08:02.492086+0000","10.20.193.151:0/1526145236":"2026-02-19T21:41:03.690123+0000","10.20.193.151:0/3983134572":"2026-02-19T21:56:39.396549+0000","10.20.193.151:0/1346373771":"2026-02-19T21:55:52.406393+0000","10.20.193.151:0/3817007402":"2026-02-19T21:51:55.959057+0000","10.20.193.151:0/207303863":"2026-02-19T21:56:54.449825+0000","10.20.193.151:0/1950716753":"2026-02-19T21:57:34.128650+0000","10.20.193.151:0/4073120043":"2026-02-19T21:57:59.957859+0000","10.20.193.151:0/4235653094":"2026-02-19T21:58:19.971856+0000","10.20.193.151:0/3432856425":"2026-02-19T21:58:29.796715+0000","10.20.193.151:0/1636048730":"2026-02-19T21:58:10.648118+0000","10.20.193.151:0/1539480227":"2026-02-19T21:58:34.270800+0000","10.20.193.151:0/82647743":"2026-02-19T22:00:58.343379+0000","10.20.193.151:0/315763429":"2026-02-19T21:58:44.803638+0000","10.20.193.151:0/2693444148":"2026-02-19T21:59:00.756185+0000","10.20.193.151:0/2063154100":"2026-02-19T21:59:09.396175+0000","10.20.193.151:0/825573180":"2026-02-19T22:02:37.023172+0000","10.20.193.151:0/3501539945":"2026-02-19T21:59:30.648851+0000","10.20.193.151:0/1031527344":"2026-02-19T21:39:50.900732+0000","10.20.193.151:0/3917665067":"2026-02-19T21:59:53.471758+0000","10.20.193.151:0/1068528727":"2026-02-19T21:59:45.189067+0000","10.20.193.151:0/2572397549":"2026-02-19T21:59:59.308207+0000","10.20.193.151:0/1040761178":"2026-02-19T22:01:10.842826+0000","10.20.193.151:0/1375281261":"2026-02-19T21:31:52.637897+0000","10.20.193.151:0/637951053":"2026-02-19T21:57:25.741409+0000","10.20.193.151:0/184041782":"2026-02-19T22:01:33.829012+0000","10.20.193.151:0/1513433874":"2026-02-19T22:02:48.902618+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T21:11:28.679 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-19T21:11:28.679 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-19T21:11:28.757 INFO:teuthology.orchestra.run.trial151.stdout:{ 2026-02-19T21:11:28.757 INFO:teuthology.orchestra.run.trial151.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-19T21:11:28.757 INFO:teuthology.orchestra.run.trial151.stdout:} 2026-02-19T21:11:28.766 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-19T21:11:28.930 INFO:teuthology.orchestra.run.trial151.stderr:instructed osd(s) 0 to deep-scrub 2026-02-19T21:11:28.942 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-19T21:11:28.942 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-19T21:11:29.018 INFO:teuthology.orchestra.run.trial151.stdout:{ 2026-02-19T21:11:29.018 INFO:teuthology.orchestra.run.trial151.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-19T21:11:29.018 INFO:teuthology.orchestra.run.trial151.stdout:} 2026-02-19T21:11:29.027 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-19T21:11:29.189 INFO:teuthology.orchestra.run.trial151.stderr:instructed osd(s) 1 to deep-scrub 2026-02-19T21:11:29.202 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-19T21:11:29.203 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-19T21:11:29.288 INFO:teuthology.orchestra.run.trial151.stdout:{ 2026-02-19T21:11:29.289 INFO:teuthology.orchestra.run.trial151.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-19T21:11:29.289 INFO:teuthology.orchestra.run.trial151.stdout:} 2026-02-19T21:11:29.298 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-19T21:11:29.461 INFO:teuthology.orchestra.run.trial151.stderr:instructed osd(s) 2 to deep-scrub 2026-02-19T21:11:29.475 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-19T21:11:29.640 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T21:11:29.640 INFO:teuthology.orchestra.run.trial151.stderr:dumped all 2026-02-19T21:11:29.653 INFO:teuthology.orchestra.run.trial151.stdout:{"pg_ready":true,"pg_map":{"version":1687,"stamp":"2026-02-19T21:11:28.902496+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":227,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4735540,"kb_used_data":15684,"kb_used_omap":1179,"kb_used_meta":4718628,"kb_avail":278379980,"statfs":{"total":289910292480,"available":285061099520,"internally_reserved":0,"allocated":16060416,"data_stored":8073950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1207463,"internal_metadata":4831875929},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-896455906,"num_objects":-198,"num_object_clones":0,"num_object_copies":-396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-198,"num_whiteouts":0,"num_read":-2369777,"num_read_kb":-90003020,"num_write":-2291455,"num_write_kb":-89006296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.961601"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":844,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.841290+0000","last_change":"2026-02-19T20:29:11.745770+0000","last_active":"2026-02-19T21:11:24.841290+0000","last_peered":"2026-02-19T21:11:24.841290+0000","last_clean":"2026-02-19T21:11:24.841290+0000","last_became_active":"2026-02-19T20:29:09.741410+0000","last_became_peered":"2026-02-19T20:29:09.741410+0000","last_unstale":"2026-02-19T21:11:24.841290+0000","last_undegraded":"2026-02-19T21:11:24.841290+0000","last_fullsized":"2026-02-19T21:11:24.841290+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:27:41.302717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052313100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":844,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.842400+0000","last_change":"2026-02-19T20:29:11.745905+0000","last_active":"2026-02-19T21:11:24.842400+0000","last_peered":"2026-02-19T21:11:24.842400+0000","last_clean":"2026-02-19T21:11:24.842400+0000","last_became_active":"2026-02-19T20:29:09.741207+0000","last_became_peered":"2026-02-19T20:29:09.741207+0000","last_unstale":"2026-02-19T21:11:24.842400+0000","last_undegraded":"2026-02-19T21:11:24.842400+0000","last_fullsized":"2026-02-19T21:11:24.842400+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:30:59.713739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066944300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":845,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.844066+0000","last_change":"2026-02-19T20:29:11.745724+0000","last_active":"2026-02-19T21:11:24.844066+0000","last_peered":"2026-02-19T21:11:24.844066+0000","last_clean":"2026-02-19T21:11:24.844066+0000","last_became_active":"2026-02-19T20:29:09.741235+0000","last_became_peered":"2026-02-19T20:29:09.741235+0000","last_unstale":"2026-02-19T21:11:24.844066+0000","last_undegraded":"2026-02-19T21:11:24.844066+0000","last_fullsized":"2026-02-19T21:11:24.844066+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:34:53.768773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051114900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":845,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.840274+0000","last_change":"2026-02-19T20:29:11.745539+0000","last_active":"2026-02-19T21:11:24.840274+0000","last_peered":"2026-02-19T21:11:24.840274+0000","last_clean":"2026-02-19T21:11:24.840274+0000","last_became_active":"2026-02-19T20:29:09.740922+0000","last_became_peered":"2026-02-19T20:29:09.740922+0000","last_unstale":"2026-02-19T21:11:24.840274+0000","last_undegraded":"2026-02-19T21:11:24.840274+0000","last_fullsized":"2026-02-19T21:11:24.840274+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:44:16.698249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353877,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":848,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.494714+0000","last_change":"2026-02-19T20:29:11.743941+0000","last_active":"2026-02-19T21:11:24.494714+0000","last_peered":"2026-02-19T21:11:24.494714+0000","last_clean":"2026-02-19T21:11:24.494714+0000","last_became_active":"2026-02-19T20:29:09.740605+0000","last_became_peered":"2026-02-19T20:29:09.740605+0000","last_unstale":"2026-02-19T21:11:24.494714+0000","last_undegraded":"2026-02-19T21:11:24.494714+0000","last_fullsized":"2026-02-19T21:11:24.494714+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T00:12:21.879309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083745499999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":848,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:25.967031+0000","last_change":"2026-02-19T20:29:11.741215+0000","last_active":"2026-02-19T21:11:25.967031+0000","last_peered":"2026-02-19T21:11:25.967031+0000","last_clean":"2026-02-19T21:11:25.967031+0000","last_became_active":"2026-02-19T20:29:09.740655+0000","last_became_peered":"2026-02-19T20:29:09.740655+0000","last_unstale":"2026-02-19T21:11:25.967031+0000","last_undegraded":"2026-02-19T21:11:25.967031+0000","last_fullsized":"2026-02-19T21:11:25.967031+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:52:18.535895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037179999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":849,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:25.965916+0000","last_change":"2026-02-19T20:29:11.741323+0000","last_active":"2026-02-19T21:11:25.965916+0000","last_peered":"2026-02-19T21:11:25.965916+0000","last_clean":"2026-02-19T21:11:25.965916+0000","last_became_active":"2026-02-19T20:29:09.740330+0000","last_became_peered":"2026-02-19T20:29:09.740330+0000","last_unstale":"2026-02-19T21:11:25.965916+0000","last_undegraded":"2026-02-19T21:11:25.965916+0000","last_fullsized":"2026-02-19T21:11:25.965916+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:54:37.429972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041288700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":846,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.843514+0000","last_change":"2026-02-19T20:29:11.745555+0000","last_active":"2026-02-19T21:11:24.843514+0000","last_peered":"2026-02-19T21:11:24.843514+0000","last_clean":"2026-02-19T21:11:24.843514+0000","last_became_active":"2026-02-19T20:29:09.740070+0000","last_became_peered":"2026-02-19T20:29:09.740070+0000","last_unstale":"2026-02-19T21:11:24.843514+0000","last_undegraded":"2026-02-19T21:11:24.843514+0000","last_fullsized":"2026-02-19T21:11:24.843514+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:08.731026+0000","last_clean_scrub_stamp":"2026-02-19T20:29:08.731026+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T22:03:17.974819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036770299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":935,"reported_epoch":417,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.842526+0000","last_change":"2026-02-19T20:29:07.943283+0000","last_active":"2026-02-19T21:11:24.842526+0000","last_peered":"2026-02-19T21:11:24.842526+0000","last_clean":"2026-02-19T21:11:24.842526+0000","last_became_active":"2026-02-19T20:29:07.942262+0000","last_became_peered":"2026-02-19T20:29:07.942262+0000","last_unstale":"2026-02-19T21:11:24.842526+0000","last_undegraded":"2026-02-19T21:11:24.842526+0000","last_fullsized":"2026-02-19T21:11:24.842526+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:29:06.726163+0000","last_clean_scrub_stamp":"2026-02-19T20:29:06.726163+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:37:51.447976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738880,"num_pgs":79,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1343872,"kb_used_data":4848,"kb_used_omap":344,"kb_used_meta":1338663,"kb_avail":93027968,"statfs":{"total":96636764160,"available":95260639232,"internally_reserved":0,"allocated":4964352,"data_stored":2302147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":352789,"internal_metadata":1370791403},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738880,"num_pgs":99,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1676604,"kb_used_data":5420,"kb_used_omap":494,"kb_used_meta":1670673,"kb_avail":92695236,"statfs":{"total":96636764160,"available":94919921664,"internally_reserved":0,"allocated":5550080,"data_stored":2885911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":506419,"internal_metadata":1710769613},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738879,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1715064,"kb_used_data":5416,"kb_used_omap":340,"kb_used_meta":1709291,"kb_avail":92656776,"statfs":{"total":96636764160,"available":94880538624,"internally_reserved":0,"allocated":5545984,"data_stored":2885892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":348255,"internal_metadata":1750314913},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T21:11:29.661 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-19T20:29:08.731026+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=29, tm_sec=8, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=11, tm_sec=27, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:11:29.661 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-19T20:29:08.731026+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=29, tm_sec=8, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=11, tm_sec=27, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:11:29.661 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-19T20:29:08.731026+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=29, tm_sec=8, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=11, tm_sec=27, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:11:29.661 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-19T20:29:08.731026+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=29, tm_sec=8, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=11, tm_sec=27, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:11:29.662 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-19T20:29:08.731026+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=29, tm_sec=8, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=11, tm_sec=27, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:11:29.662 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-19T20:29:08.731026+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=29, tm_sec=8, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=11, tm_sec=27, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:11:29.662 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-19T20:29:08.731026+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=29, tm_sec=8, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=11, tm_sec=27, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:11:29.662 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-19T20:29:08.731026+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=29, tm_sec=8, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=11, tm_sec=27, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:11:29.662 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-19T20:29:06.726163+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=29, tm_sec=6, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=11, tm_sec=27, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:11:29.662 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-19T21:11:49.664 DEBUG:teuthology.orchestra.run.trial151:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-19T21:11:49.826 INFO:teuthology.orchestra.run.trial151.stdout: 2026-02-19T21:11:49.827 INFO:teuthology.orchestra.run.trial151.stderr:dumped all 2026-02-19T21:11:49.836 INFO:teuthology.orchestra.run.trial151.stdout:{"pg_ready":true,"pg_map":{"version":1697,"stamp":"2026-02-19T21:11:48.904266+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4735568,"kb_used_data":15712,"kb_used_omap":1182,"kb_used_meta":4718625,"kb_avail":278379952,"statfs":{"total":289910292480,"available":285061070848,"internally_reserved":0,"allocated":16089088,"data_stored":8087215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1210951,"internal_metadata":4831872441},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001085"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":854,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:34.722954+0000","last_change":"2026-02-19T21:11:34.722954+0000","last_active":"2026-02-19T21:11:34.722954+0000","last_peered":"2026-02-19T21:11:34.722954+0000","last_clean":"2026-02-19T21:11:34.722954+0000","last_became_active":"2026-02-19T20:29:09.741410+0000","last_became_peered":"2026-02-19T20:29:09.741410+0000","last_unstale":"2026-02-19T21:11:34.722954+0000","last_undegraded":"2026-02-19T21:11:34.722954+0000","last_fullsized":"2026-02-19T21:11:34.722954+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:11:34.722909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:11:34.722909+0000","last_clean_scrub_stamp":"2026-02-19T21:11:34.722909+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T23:52:40.923511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052313100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":854,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:32.785335+0000","last_change":"2026-02-19T21:11:32.785335+0000","last_active":"2026-02-19T21:11:32.785335+0000","last_peered":"2026-02-19T21:11:32.785335+0000","last_clean":"2026-02-19T21:11:32.785335+0000","last_became_active":"2026-02-19T20:29:09.741207+0000","last_became_peered":"2026-02-19T20:29:09.741207+0000","last_unstale":"2026-02-19T21:11:32.785335+0000","last_undegraded":"2026-02-19T21:11:32.785335+0000","last_fullsized":"2026-02-19T21:11:32.785335+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:11:32.785271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:11:32.785271+0000","last_clean_scrub_stamp":"2026-02-19T21:11:32.785271+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T01:29:11.161741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066944300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":855,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:33.741109+0000","last_change":"2026-02-19T21:11:33.741109+0000","last_active":"2026-02-19T21:11:33.741109+0000","last_peered":"2026-02-19T21:11:33.741109+0000","last_clean":"2026-02-19T21:11:33.741109+0000","last_became_active":"2026-02-19T20:29:09.741235+0000","last_became_peered":"2026-02-19T20:29:09.741235+0000","last_unstale":"2026-02-19T21:11:33.741109+0000","last_undegraded":"2026-02-19T21:11:33.741109+0000","last_fullsized":"2026-02-19T21:11:33.741109+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:11:33.741064+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:11:33.741064+0000","last_clean_scrub_stamp":"2026-02-19T21:11:33.741064+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T05:43:09.204156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051114900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":855,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:29.807331+0000","last_change":"2026-02-19T21:11:29.807331+0000","last_active":"2026-02-19T21:11:29.807331+0000","last_peered":"2026-02-19T21:11:29.807331+0000","last_clean":"2026-02-19T21:11:29.807331+0000","last_became_active":"2026-02-19T20:29:09.740922+0000","last_became_peered":"2026-02-19T20:29:09.740922+0000","last_unstale":"2026-02-19T21:11:29.807331+0000","last_undegraded":"2026-02-19T21:11:29.807331+0000","last_fullsized":"2026-02-19T21:11:29.807331+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:11:29.807258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:11:29.807258+0000","last_clean_scrub_stamp":"2026-02-19T21:11:29.807258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:23:47.511488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353877,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"418'3","reported_seq":859,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:29.765978+0000","last_change":"2026-02-19T21:11:29.765978+0000","last_active":"2026-02-19T21:11:29.765978+0000","last_peered":"2026-02-19T21:11:29.765978+0000","last_clean":"2026-02-19T21:11:29.765978+0000","last_became_active":"2026-02-19T20:29:09.740605+0000","last_became_peered":"2026-02-19T20:29:09.740605+0000","last_unstale":"2026-02-19T21:11:29.765978+0000","last_undegraded":"2026-02-19T21:11:29.765978+0000","last_fullsized":"2026-02-19T21:11:29.765978+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"418'3","last_scrub_stamp":"2026-02-19T21:11:29.765907+0000","last_deep_scrub":"418'3","last_deep_scrub_stamp":"2026-02-19T21:11:29.765907+0000","last_clean_scrub_stamp":"2026-02-19T21:11:29.765907+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T23:41:27.127630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083745499999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":856,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:31.191161+0000","last_change":"2026-02-19T21:11:31.191161+0000","last_active":"2026-02-19T21:11:31.191161+0000","last_peered":"2026-02-19T21:11:31.191161+0000","last_clean":"2026-02-19T21:11:31.191161+0000","last_became_active":"2026-02-19T20:29:09.740655+0000","last_became_peered":"2026-02-19T20:29:09.740655+0000","last_unstale":"2026-02-19T21:11:31.191161+0000","last_undegraded":"2026-02-19T21:11:31.191161+0000","last_fullsized":"2026-02-19T21:11:31.191161+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:11:31.191078+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:11:31.191078+0000","last_clean_scrub_stamp":"2026-02-19T21:11:31.191078+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T23:37:21.934545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037179999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":857,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:30.141858+0000","last_change":"2026-02-19T21:11:30.141858+0000","last_active":"2026-02-19T21:11:30.141858+0000","last_peered":"2026-02-19T21:11:30.141858+0000","last_clean":"2026-02-19T21:11:30.141858+0000","last_became_active":"2026-02-19T20:29:09.740330+0000","last_became_peered":"2026-02-19T20:29:09.740330+0000","last_unstale":"2026-02-19T21:11:30.141858+0000","last_undegraded":"2026-02-19T21:11:30.141858+0000","last_fullsized":"2026-02-19T21:11:30.141858+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:11:30.141792+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:11:30.141792+0000","last_clean_scrub_stamp":"2026-02-19T21:11:30.141792+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T08:44:12.656547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041288700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"418'2","reported_seq":857,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:30.816815+0000","last_change":"2026-02-19T21:11:30.816815+0000","last_active":"2026-02-19T21:11:30.816815+0000","last_peered":"2026-02-19T21:11:30.816815+0000","last_clean":"2026-02-19T21:11:30.816815+0000","last_became_active":"2026-02-19T20:29:09.740070+0000","last_became_peered":"2026-02-19T20:29:09.740070+0000","last_unstale":"2026-02-19T21:11:30.816815+0000","last_undegraded":"2026-02-19T21:11:30.816815+0000","last_fullsized":"2026-02-19T21:11:30.816815+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"418'2","last_scrub_stamp":"2026-02-19T21:11:30.816775+0000","last_deep_scrub":"418'2","last_deep_scrub_stamp":"2026-02-19T21:11:30.816775+0000","last_clean_scrub_stamp":"2026-02-19T21:11:30.816775+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:45:56.216517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036770299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"418'74","reported_seq":947,"reported_epoch":418,"state":"active+clean","last_fresh":"2026-02-19T21:11:31.789272+0000","last_change":"2026-02-19T21:11:31.789272+0000","last_active":"2026-02-19T21:11:31.789272+0000","last_peered":"2026-02-19T21:11:31.789272+0000","last_clean":"2026-02-19T21:11:31.789272+0000","last_became_active":"2026-02-19T20:29:07.942262+0000","last_became_peered":"2026-02-19T20:29:07.942262+0000","last_unstale":"2026-02-19T21:11:31.789272+0000","last_undegraded":"2026-02-19T21:11:31.789272+0000","last_fullsized":"2026-02-19T21:11:31.789272+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"418'74","last_scrub_stamp":"2026-02-19T21:11:31.789215+0000","last_deep_scrub":"418'74","last_deep_scrub_stamp":"2026-02-19T21:11:31.789215+0000","last_clean_scrub_stamp":"2026-02-19T21:11:31.789215+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T03:13:22.751488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738884,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1343872,"kb_used_data":4848,"kb_used_omap":344,"kb_used_meta":1338663,"kb_avail":93027968,"statfs":{"total":96636764160,"available":95260639232,"internally_reserved":0,"allocated":4964352,"data_stored":2302147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":352789,"internal_metadata":1370791403},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738884,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1676616,"kb_used_data":5432,"kb_used_omap":496,"kb_used_meta":1670671,"kb_avail":92695224,"statfs":{"total":96636764160,"available":94919909376,"internally_reserved":0,"allocated":5562368,"data_stored":2892534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":508565,"internal_metadata":1710767467},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738883,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1715080,"kb_used_data":5432,"kb_used_omap":341,"kb_used_meta":1709290,"kb_avail":92656760,"statfs":{"total":96636764160,"available":94880522240,"internally_reserved":0,"allocated":5562368,"data_stored":2892534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":349597,"internal_metadata":1750313571},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T21:11:49.837 DEBUG:teuthology.orchestra.run.trial151:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-19T21:11:50.002 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-19T21:11:50.003 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-19T21:11:50.003 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-19T21:11:50.003 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:11:51.540 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~0s 2026-02-19T21:11:56.105 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-19T21:11:56.105 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-19T21:11:56.106 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:11:56.843 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.1 is failed for ~0s 2026-02-19T21:11:56.843 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 has been restored 2026-02-19T21:12:02.146 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.1 is failed for ~5s 2026-02-19T21:12:02.208 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-19T21:12:02.208 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-19T21:12:02.208 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:12:07.348 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 is failed for ~0s 2026-02-19T21:12:07.348 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.1 has been restored 2026-02-19T21:12:08.310 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-19T21:12:08.310 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-19T21:12:08.310 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-19T21:12:08.310 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:12:08.362 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-19T21:12:08.362 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-19T21:12:08.362 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-19T21:12:08.362 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:12:08.382 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-19T21:12:08.382 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-19T21:12:08.382 DEBUG:teuthology.orchestra.run.trial151:> sudo grep -E '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | grep -E -v '\(MDS_ALL_DOWN\)' | grep -E -v '\(MDS_UP_LESS_THAN_MAX\)' | grep -E -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-19T21:12:08.434 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial151.front.sepia.ceph.com 2026-02-19T21:12:08.435 DEBUG:teuthology.orchestra.run.trial151:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-19T21:12:08.891 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial151.front.sepia.ceph.com 2026-02-19T21:12:08.891 DEBUG:teuthology.orchestra.run.trial151:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-19T21:12:09.304 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial151.front.sepia.ceph.com 2026-02-19T21:12:09.304 DEBUG:teuthology.orchestra.run.trial151:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-19T21:12:09.751 INFO:tasks.ceph:Archiving mon data... 2026-02-19T21:12:09.752 DEBUG:teuthology.misc:Transferring archived files from trial151:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645/data/cluster2.mon.a.tgz 2026-02-19T21:12:09.752 DEBUG:teuthology.orchestra.run.trial151:> mktemp 2026-02-19T21:12:09.756 INFO:teuthology.orchestra.run.trial151.stdout:/tmp/tmp.pM7WL6lO02 2026-02-19T21:12:09.757 DEBUG:teuthology.orchestra.run.trial151:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.pM7WL6lO02 2026-02-19T21:12:09.854 DEBUG:teuthology.orchestra.run.trial151:> sudo chmod 0666 /tmp/tmp.pM7WL6lO02 2026-02-19T21:12:09.909 DEBUG:teuthology.orchestra.remote:trial151:/tmp/tmp.pM7WL6lO02 is 417KB 2026-02-19T21:12:09.975 DEBUG:teuthology.orchestra.run.trial151:> rm -fr /tmp/tmp.pM7WL6lO02 2026-02-19T21:12:09.979 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-19T21:12:09.979 DEBUG:teuthology.orchestra.run.trial013:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-19T21:12:09.982 DEBUG:teuthology.orchestra.run.trial151:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-19T21:12:10.029 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-19T21:12:10.048 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-19T21:12:10.049 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-19T21:12:10.199 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T21:12:10.200 INFO:teuthology.orchestra.run.trial013.stderr:dumped all 2026-02-19T21:12:10.211 INFO:teuthology.orchestra.run.trial013.stdout:{"pg_ready":true,"pg_map":{"version":1644,"stamp":"2026-02-19T21:12:09.291725+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3922072,"kb_used_data":9448,"kb_used_omap":2638,"kb_used_meta":3909937,"kb_avail":279193448,"statfs":{"total":289910292480,"available":285894090752,"internally_reserved":0,"allocated":9674752,"data_stored":5664349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2701479,"internal_metadata":4003776345},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000931"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":696,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.897234+0000","last_change":"2026-02-19T20:28:49.932388+0000","last_active":"2026-02-19T21:11:24.897234+0000","last_peered":"2026-02-19T21:11:24.897234+0000","last_clean":"2026-02-19T21:11:24.897234+0000","last_became_active":"2026-02-19T20:28:47.929308+0000","last_became_peered":"2026-02-19T20:28:47.929308+0000","last_unstale":"2026-02-19T21:11:24.897234+0000","last_undegraded":"2026-02-19T21:11:24.897234+0000","last_fullsized":"2026-02-19T21:11:24.897234+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:51:59.644924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086370499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.899360+0000","last_change":"2026-02-19T20:28:49.932057+0000","last_active":"2026-02-19T21:11:24.899360+0000","last_peered":"2026-02-19T21:11:24.899360+0000","last_clean":"2026-02-19T21:11:24.899360+0000","last_became_active":"2026-02-19T20:28:47.929421+0000","last_became_peered":"2026-02-19T20:28:47.929421+0000","last_unstale":"2026-02-19T21:11:24.899360+0000","last_undegraded":"2026-02-19T21:11:24.899360+0000","last_fullsized":"2026-02-19T21:11:24.899360+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:53:36.261502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055284699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.899354+0000","last_change":"2026-02-19T20:28:49.932501+0000","last_active":"2026-02-19T21:11:24.899354+0000","last_peered":"2026-02-19T21:11:24.899354+0000","last_clean":"2026-02-19T21:11:24.899354+0000","last_became_active":"2026-02-19T20:28:47.929096+0000","last_became_peered":"2026-02-19T20:28:47.929096+0000","last_unstale":"2026-02-19T21:11:24.899354+0000","last_undegraded":"2026-02-19T21:11:24.899354+0000","last_fullsized":"2026-02-19T21:11:24.899354+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:35:27.823970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005578,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.895037+0000","last_change":"2026-02-19T20:28:49.932359+0000","last_active":"2026-02-19T21:11:24.895037+0000","last_peered":"2026-02-19T21:11:24.895037+0000","last_clean":"2026-02-19T21:11:24.895037+0000","last_became_active":"2026-02-19T20:28:47.929286+0000","last_became_peered":"2026-02-19T20:28:47.929286+0000","last_unstale":"2026-02-19T21:11:24.895037+0000","last_undegraded":"2026-02-19T21:11:24.895037+0000","last_fullsized":"2026-02-19T21:11:24.895037+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:15:05.977395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084824599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":700,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.868750+0000","last_change":"2026-02-19T20:28:49.930433+0000","last_active":"2026-02-19T21:11:24.868750+0000","last_peered":"2026-02-19T21:11:24.868750+0000","last_clean":"2026-02-19T21:11:24.868750+0000","last_became_active":"2026-02-19T20:28:47.929119+0000","last_became_peered":"2026-02-19T20:28:47.929119+0000","last_unstale":"2026-02-19T21:11:24.868750+0000","last_undegraded":"2026-02-19T21:11:24.868750+0000","last_fullsized":"2026-02-19T21:11:24.868750+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:02:22.704407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092100199999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":698,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.894576+0000","last_change":"2026-02-19T20:28:50.333526+0000","last_active":"2026-02-19T21:11:24.894576+0000","last_peered":"2026-02-19T21:11:24.894576+0000","last_clean":"2026-02-19T21:11:24.894576+0000","last_became_active":"2026-02-19T20:28:47.929462+0000","last_became_peered":"2026-02-19T20:28:47.929462+0000","last_unstale":"2026-02-19T21:11:24.894576+0000","last_undegraded":"2026-02-19T21:11:24.894576+0000","last_fullsized":"2026-02-19T21:11:24.894576+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:14:47.461186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024374299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":699,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.894122+0000","last_change":"2026-02-19T20:28:50.333516+0000","last_active":"2026-02-19T21:11:24.894122+0000","last_peered":"2026-02-19T21:11:24.894122+0000","last_clean":"2026-02-19T21:11:24.894122+0000","last_became_active":"2026-02-19T20:28:47.929487+0000","last_became_peered":"2026-02-19T20:28:47.929487+0000","last_unstale":"2026-02-19T21:11:24.894122+0000","last_undegraded":"2026-02-19T21:11:24.894122+0000","last_fullsized":"2026-02-19T21:11:24.894122+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T03:04:05.217623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024657800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":698,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.897258+0000","last_change":"2026-02-19T20:28:49.932091+0000","last_active":"2026-02-19T21:11:24.897258+0000","last_peered":"2026-02-19T21:11:24.897258+0000","last_clean":"2026-02-19T21:11:24.897258+0000","last_became_active":"2026-02-19T20:28:47.928693+0000","last_became_peered":"2026-02-19T20:28:47.928693+0000","last_unstale":"2026-02-19T21:11:24.897258+0000","last_undegraded":"2026-02-19T21:11:24.897258+0000","last_fullsized":"2026-02-19T21:11:24.897258+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:23:54.434881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060120699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":787,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.895975+0000","last_change":"2026-02-19T20:28:45.921802+0000","last_active":"2026-02-19T21:11:24.895975+0000","last_peered":"2026-02-19T21:11:24.895975+0000","last_clean":"2026-02-19T21:11:24.895975+0000","last_became_active":"2026-02-19T20:28:45.920826+0000","last_became_peered":"2026-02-19T20:28:45.920826+0000","last_unstale":"2026-02-19T21:11:24.895975+0000","last_undegraded":"2026-02-19T21:11:24.895975+0000","last_fullsized":"2026-02-19T21:11:24.895975+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_clean_scrub_stamp":"2026-02-19T20:28:44.914354+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:32:41.646846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738891,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":511256,"kb_used_data":2760,"kb_used_omap":1385,"kb_used_meta":507094,"kb_avail":93860584,"statfs":{"total":96636764160,"available":96113238016,"internally_reserved":0,"allocated":2826240,"data_stored":1494525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1418428,"internal_metadata":519265092},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738893,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1413856,"kb_used_data":3344,"kb_used_omap":577,"kb_used_meta":1409918,"kb_avail":92957984,"statfs":{"total":96636764160,"available":95188975616,"internally_reserved":0,"allocated":3424256,"data_stored":2084912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":591262,"internal_metadata":1443756642},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738892,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1996960,"kb_used_data":3344,"kb_used_omap":675,"kb_used_meta":1992924,"kb_avail":92374880,"statfs":{"total":96636764160,"available":94591877120,"internally_reserved":0,"allocated":3424256,"data_stored":2084912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":691789,"internal_metadata":2040754611},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T21:12:10.212 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-19T21:12:10.372 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T21:12:10.372 INFO:teuthology.orchestra.run.trial013.stderr:dumped all 2026-02-19T21:12:10.383 INFO:teuthology.orchestra.run.trial013.stdout:{"pg_ready":true,"pg_map":{"version":1644,"stamp":"2026-02-19T21:12:09.291725+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3922072,"kb_used_data":9448,"kb_used_omap":2638,"kb_used_meta":3909937,"kb_avail":279193448,"statfs":{"total":289910292480,"available":285894090752,"internally_reserved":0,"allocated":9674752,"data_stored":5664349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2701479,"internal_metadata":4003776345},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000931"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":696,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.897234+0000","last_change":"2026-02-19T20:28:49.932388+0000","last_active":"2026-02-19T21:11:24.897234+0000","last_peered":"2026-02-19T21:11:24.897234+0000","last_clean":"2026-02-19T21:11:24.897234+0000","last_became_active":"2026-02-19T20:28:47.929308+0000","last_became_peered":"2026-02-19T20:28:47.929308+0000","last_unstale":"2026-02-19T21:11:24.897234+0000","last_undegraded":"2026-02-19T21:11:24.897234+0000","last_fullsized":"2026-02-19T21:11:24.897234+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:51:59.644924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086370499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.899360+0000","last_change":"2026-02-19T20:28:49.932057+0000","last_active":"2026-02-19T21:11:24.899360+0000","last_peered":"2026-02-19T21:11:24.899360+0000","last_clean":"2026-02-19T21:11:24.899360+0000","last_became_active":"2026-02-19T20:28:47.929421+0000","last_became_peered":"2026-02-19T20:28:47.929421+0000","last_unstale":"2026-02-19T21:11:24.899360+0000","last_undegraded":"2026-02-19T21:11:24.899360+0000","last_fullsized":"2026-02-19T21:11:24.899360+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:53:36.261502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055284699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.899354+0000","last_change":"2026-02-19T20:28:49.932501+0000","last_active":"2026-02-19T21:11:24.899354+0000","last_peered":"2026-02-19T21:11:24.899354+0000","last_clean":"2026-02-19T21:11:24.899354+0000","last_became_active":"2026-02-19T20:28:47.929096+0000","last_became_peered":"2026-02-19T20:28:47.929096+0000","last_unstale":"2026-02-19T21:11:24.899354+0000","last_undegraded":"2026-02-19T21:11:24.899354+0000","last_fullsized":"2026-02-19T21:11:24.899354+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:35:27.823970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005578,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.895037+0000","last_change":"2026-02-19T20:28:49.932359+0000","last_active":"2026-02-19T21:11:24.895037+0000","last_peered":"2026-02-19T21:11:24.895037+0000","last_clean":"2026-02-19T21:11:24.895037+0000","last_became_active":"2026-02-19T20:28:47.929286+0000","last_became_peered":"2026-02-19T20:28:47.929286+0000","last_unstale":"2026-02-19T21:11:24.895037+0000","last_undegraded":"2026-02-19T21:11:24.895037+0000","last_fullsized":"2026-02-19T21:11:24.895037+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:15:05.977395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084824599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":700,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.868750+0000","last_change":"2026-02-19T20:28:49.930433+0000","last_active":"2026-02-19T21:11:24.868750+0000","last_peered":"2026-02-19T21:11:24.868750+0000","last_clean":"2026-02-19T21:11:24.868750+0000","last_became_active":"2026-02-19T20:28:47.929119+0000","last_became_peered":"2026-02-19T20:28:47.929119+0000","last_unstale":"2026-02-19T21:11:24.868750+0000","last_undegraded":"2026-02-19T21:11:24.868750+0000","last_fullsized":"2026-02-19T21:11:24.868750+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:02:22.704407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092100199999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":698,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.894576+0000","last_change":"2026-02-19T20:28:50.333526+0000","last_active":"2026-02-19T21:11:24.894576+0000","last_peered":"2026-02-19T21:11:24.894576+0000","last_clean":"2026-02-19T21:11:24.894576+0000","last_became_active":"2026-02-19T20:28:47.929462+0000","last_became_peered":"2026-02-19T20:28:47.929462+0000","last_unstale":"2026-02-19T21:11:24.894576+0000","last_undegraded":"2026-02-19T21:11:24.894576+0000","last_fullsized":"2026-02-19T21:11:24.894576+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:14:47.461186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024374299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":699,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.894122+0000","last_change":"2026-02-19T20:28:50.333516+0000","last_active":"2026-02-19T21:11:24.894122+0000","last_peered":"2026-02-19T21:11:24.894122+0000","last_clean":"2026-02-19T21:11:24.894122+0000","last_became_active":"2026-02-19T20:28:47.929487+0000","last_became_peered":"2026-02-19T20:28:47.929487+0000","last_unstale":"2026-02-19T21:11:24.894122+0000","last_undegraded":"2026-02-19T21:11:24.894122+0000","last_fullsized":"2026-02-19T21:11:24.894122+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T03:04:05.217623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024657800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":698,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.897258+0000","last_change":"2026-02-19T20:28:49.932091+0000","last_active":"2026-02-19T21:11:24.897258+0000","last_peered":"2026-02-19T21:11:24.897258+0000","last_clean":"2026-02-19T21:11:24.897258+0000","last_became_active":"2026-02-19T20:28:47.928693+0000","last_became_peered":"2026-02-19T20:28:47.928693+0000","last_unstale":"2026-02-19T21:11:24.897258+0000","last_undegraded":"2026-02-19T21:11:24.897258+0000","last_fullsized":"2026-02-19T21:11:24.897258+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:23:54.434881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060120699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":787,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.895975+0000","last_change":"2026-02-19T20:28:45.921802+0000","last_active":"2026-02-19T21:11:24.895975+0000","last_peered":"2026-02-19T21:11:24.895975+0000","last_clean":"2026-02-19T21:11:24.895975+0000","last_became_active":"2026-02-19T20:28:45.920826+0000","last_became_peered":"2026-02-19T20:28:45.920826+0000","last_unstale":"2026-02-19T21:11:24.895975+0000","last_undegraded":"2026-02-19T21:11:24.895975+0000","last_fullsized":"2026-02-19T21:11:24.895975+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_clean_scrub_stamp":"2026-02-19T20:28:44.914354+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:32:41.646846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738891,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":511256,"kb_used_data":2760,"kb_used_omap":1385,"kb_used_meta":507094,"kb_avail":93860584,"statfs":{"total":96636764160,"available":96113238016,"internally_reserved":0,"allocated":2826240,"data_stored":1494525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1418428,"internal_metadata":519265092},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738893,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1413856,"kb_used_data":3344,"kb_used_omap":577,"kb_used_meta":1409918,"kb_avail":92957984,"statfs":{"total":96636764160,"available":95188975616,"internally_reserved":0,"allocated":3424256,"data_stored":2084912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":591262,"internal_metadata":1443756642},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738892,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1996960,"kb_used_data":3344,"kb_used_omap":675,"kb_used_meta":1992924,"kb_avail":92374880,"statfs":{"total":96636764160,"available":94591877120,"internally_reserved":0,"allocated":3424256,"data_stored":2084912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":691789,"internal_metadata":2040754611},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T21:12:10.383 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-19T21:12:10.383 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-19T21:12:10.542 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T21:12:10.543 INFO:teuthology.orchestra.run.trial013.stderr:dumped all 2026-02-19T21:12:10.553 INFO:teuthology.orchestra.run.trial013.stdout:{"pg_ready":true,"pg_map":{"version":1644,"stamp":"2026-02-19T21:12:09.291725+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3922072,"kb_used_data":9448,"kb_used_omap":2638,"kb_used_meta":3909937,"kb_avail":279193448,"statfs":{"total":289910292480,"available":285894090752,"internally_reserved":0,"allocated":9674752,"data_stored":5664349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2701479,"internal_metadata":4003776345},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000931"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":696,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.897234+0000","last_change":"2026-02-19T20:28:49.932388+0000","last_active":"2026-02-19T21:11:24.897234+0000","last_peered":"2026-02-19T21:11:24.897234+0000","last_clean":"2026-02-19T21:11:24.897234+0000","last_became_active":"2026-02-19T20:28:47.929308+0000","last_became_peered":"2026-02-19T20:28:47.929308+0000","last_unstale":"2026-02-19T21:11:24.897234+0000","last_undegraded":"2026-02-19T21:11:24.897234+0000","last_fullsized":"2026-02-19T21:11:24.897234+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:51:59.644924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086370499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.899360+0000","last_change":"2026-02-19T20:28:49.932057+0000","last_active":"2026-02-19T21:11:24.899360+0000","last_peered":"2026-02-19T21:11:24.899360+0000","last_clean":"2026-02-19T21:11:24.899360+0000","last_became_active":"2026-02-19T20:28:47.929421+0000","last_became_peered":"2026-02-19T20:28:47.929421+0000","last_unstale":"2026-02-19T21:11:24.899360+0000","last_undegraded":"2026-02-19T21:11:24.899360+0000","last_fullsized":"2026-02-19T21:11:24.899360+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:53:36.261502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055284699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.899354+0000","last_change":"2026-02-19T20:28:49.932501+0000","last_active":"2026-02-19T21:11:24.899354+0000","last_peered":"2026-02-19T21:11:24.899354+0000","last_clean":"2026-02-19T21:11:24.899354+0000","last_became_active":"2026-02-19T20:28:47.929096+0000","last_became_peered":"2026-02-19T20:28:47.929096+0000","last_unstale":"2026-02-19T21:11:24.899354+0000","last_undegraded":"2026-02-19T21:11:24.899354+0000","last_fullsized":"2026-02-19T21:11:24.899354+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:35:27.823970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005578,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.895037+0000","last_change":"2026-02-19T20:28:49.932359+0000","last_active":"2026-02-19T21:11:24.895037+0000","last_peered":"2026-02-19T21:11:24.895037+0000","last_clean":"2026-02-19T21:11:24.895037+0000","last_became_active":"2026-02-19T20:28:47.929286+0000","last_became_peered":"2026-02-19T20:28:47.929286+0000","last_unstale":"2026-02-19T21:11:24.895037+0000","last_undegraded":"2026-02-19T21:11:24.895037+0000","last_fullsized":"2026-02-19T21:11:24.895037+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:15:05.977395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084824599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":700,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.868750+0000","last_change":"2026-02-19T20:28:49.930433+0000","last_active":"2026-02-19T21:11:24.868750+0000","last_peered":"2026-02-19T21:11:24.868750+0000","last_clean":"2026-02-19T21:11:24.868750+0000","last_became_active":"2026-02-19T20:28:47.929119+0000","last_became_peered":"2026-02-19T20:28:47.929119+0000","last_unstale":"2026-02-19T21:11:24.868750+0000","last_undegraded":"2026-02-19T21:11:24.868750+0000","last_fullsized":"2026-02-19T21:11:24.868750+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:02:22.704407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092100199999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":698,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.894576+0000","last_change":"2026-02-19T20:28:50.333526+0000","last_active":"2026-02-19T21:11:24.894576+0000","last_peered":"2026-02-19T21:11:24.894576+0000","last_clean":"2026-02-19T21:11:24.894576+0000","last_became_active":"2026-02-19T20:28:47.929462+0000","last_became_peered":"2026-02-19T20:28:47.929462+0000","last_unstale":"2026-02-19T21:11:24.894576+0000","last_undegraded":"2026-02-19T21:11:24.894576+0000","last_fullsized":"2026-02-19T21:11:24.894576+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:14:47.461186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024374299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":699,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.894122+0000","last_change":"2026-02-19T20:28:50.333516+0000","last_active":"2026-02-19T21:11:24.894122+0000","last_peered":"2026-02-19T21:11:24.894122+0000","last_clean":"2026-02-19T21:11:24.894122+0000","last_became_active":"2026-02-19T20:28:47.929487+0000","last_became_peered":"2026-02-19T20:28:47.929487+0000","last_unstale":"2026-02-19T21:11:24.894122+0000","last_undegraded":"2026-02-19T21:11:24.894122+0000","last_fullsized":"2026-02-19T21:11:24.894122+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T03:04:05.217623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024657800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":698,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.897258+0000","last_change":"2026-02-19T20:28:49.932091+0000","last_active":"2026-02-19T21:11:24.897258+0000","last_peered":"2026-02-19T21:11:24.897258+0000","last_clean":"2026-02-19T21:11:24.897258+0000","last_became_active":"2026-02-19T20:28:47.928693+0000","last_became_peered":"2026-02-19T20:28:47.928693+0000","last_unstale":"2026-02-19T21:11:24.897258+0000","last_undegraded":"2026-02-19T21:11:24.897258+0000","last_fullsized":"2026-02-19T21:11:24.897258+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:23:54.434881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060120699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":787,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.895975+0000","last_change":"2026-02-19T20:28:45.921802+0000","last_active":"2026-02-19T21:11:24.895975+0000","last_peered":"2026-02-19T21:11:24.895975+0000","last_clean":"2026-02-19T21:11:24.895975+0000","last_became_active":"2026-02-19T20:28:45.920826+0000","last_became_peered":"2026-02-19T20:28:45.920826+0000","last_unstale":"2026-02-19T21:11:24.895975+0000","last_undegraded":"2026-02-19T21:11:24.895975+0000","last_fullsized":"2026-02-19T21:11:24.895975+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_clean_scrub_stamp":"2026-02-19T20:28:44.914354+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:32:41.646846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738891,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":511256,"kb_used_data":2760,"kb_used_omap":1385,"kb_used_meta":507094,"kb_avail":93860584,"statfs":{"total":96636764160,"available":96113238016,"internally_reserved":0,"allocated":2826240,"data_stored":1494525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1418428,"internal_metadata":519265092},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738893,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1413856,"kb_used_data":3344,"kb_used_omap":577,"kb_used_meta":1409918,"kb_avail":92957984,"statfs":{"total":96636764160,"available":95188975616,"internally_reserved":0,"allocated":3424256,"data_stored":2084912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":591262,"internal_metadata":1443756642},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738892,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1996960,"kb_used_data":3344,"kb_used_omap":675,"kb_used_meta":1992924,"kb_avail":92374880,"statfs":{"total":96636764160,"available":94591877120,"internally_reserved":0,"allocated":3424256,"data_stored":2084912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":691789,"internal_metadata":2040754611},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T21:12:10.554 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-19T21:12:10.705 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T21:12:10.706 INFO:teuthology.orchestra.run.trial013.stdout:{"epoch":343,"fsid":"ce984b27-e607-434f-a544-5b69d6ba521e","created":"2026-02-19T20:28:38.641475+0000","modified":"2026-02-19T21:11:24.862245+0000","last_up_change":"2026-02-19T20:28:43.886354+0000","last_in_change":"2026-02-19T20:28:39.256107+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-19T20:28:44.883525+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-19T20:28:46.611861+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b21c7b36-5ce8-4233-a9f0-ba75115b2697","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6800","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6801","nonce":3970146534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6802","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6803","nonce":3970146534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6806","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6807","nonce":3970146534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6804","nonce":3970146534},{"type":"v1","addr":"10.20.193.13:6805","nonce":3970146534}]},"public_addr":"10.20.193.13:6801/3970146534","cluster_addr":"10.20.193.13:6803/3970146534","heartbeat_back_addr":"10.20.193.13:6807/3970146534","heartbeat_front_addr":"10.20.193.13:6805/3970146534","state":["exists","up"]},{"osd":1,"uuid":"c35eb6ed-9cea-4110-8e0d-078ee3016838","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6810","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6811","nonce":61555766}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6813","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6816","nonce":61555766}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6822","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6823","nonce":61555766}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6818","nonce":61555766},{"type":"v1","addr":"10.20.193.13:6820","nonce":61555766}]},"public_addr":"10.20.193.13:6811/61555766","cluster_addr":"10.20.193.13:6816/61555766","heartbeat_back_addr":"10.20.193.13:6823/61555766","heartbeat_front_addr":"10.20.193.13:6820/61555766","state":["exists","up"]},{"osd":2,"uuid":"16338c81-9245-466a-98d3-99bcc7dbf2b6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6808","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6809","nonce":2806143231}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6812","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6814","nonce":2806143231}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6819","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6821","nonce":2806143231}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.13:6815","nonce":2806143231},{"type":"v1","addr":"10.20.193.13:6817","nonce":2806143231}]},"public_addr":"10.20.193.13:6809/2806143231","cluster_addr":"10.20.193.13:6814/2806143231","heartbeat_back_addr":"10.20.193.13:6821/2806143231","heartbeat_front_addr":"10.20.193.13:6817/2806143231","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-19T20:28:42.314257+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-19T20:28:42.345287+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-19T20:28:42.257304+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-19T21:12:11.718 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-19T21:12:11.718 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-19T21:12:11.804 INFO:teuthology.orchestra.run.trial013.stdout:{ 2026-02-19T21:12:11.804 INFO:teuthology.orchestra.run.trial013.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-19T21:12:11.804 INFO:teuthology.orchestra.run.trial013.stdout:} 2026-02-19T21:12:11.814 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-19T21:12:11.972 INFO:teuthology.orchestra.run.trial013.stderr:instructed osd(s) 0 to deep-scrub 2026-02-19T21:12:11.984 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-19T21:12:11.984 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-19T21:12:12.059 INFO:teuthology.orchestra.run.trial013.stdout:{ 2026-02-19T21:12:12.059 INFO:teuthology.orchestra.run.trial013.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-19T21:12:12.059 INFO:teuthology.orchestra.run.trial013.stdout:} 2026-02-19T21:12:12.068 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-19T21:12:12.224 INFO:teuthology.orchestra.run.trial013.stderr:instructed osd(s) 1 to deep-scrub 2026-02-19T21:12:12.236 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-19T21:12:12.236 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-19T21:12:12.313 INFO:teuthology.orchestra.run.trial013.stdout:{ 2026-02-19T21:12:12.313 INFO:teuthology.orchestra.run.trial013.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-19T21:12:12.313 INFO:teuthology.orchestra.run.trial013.stdout:} 2026-02-19T21:12:12.326 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-19T21:12:12.349 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 has been restored 2026-02-19T21:12:12.479 INFO:teuthology.orchestra.run.trial013.stderr:instructed osd(s) 2 to deep-scrub 2026-02-19T21:12:12.491 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-19T21:12:12.643 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T21:12:12.644 INFO:teuthology.orchestra.run.trial013.stderr:dumped all 2026-02-19T21:12:12.654 INFO:teuthology.orchestra.run.trial013.stdout:{"pg_ready":true,"pg_map":{"version":1645,"stamp":"2026-02-19T21:12:11.291914+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3922072,"kb_used_data":9448,"kb_used_omap":2638,"kb_used_meta":3909937,"kb_avail":279193448,"statfs":{"total":289910292480,"available":285894090752,"internally_reserved":0,"allocated":9674752,"data_stored":5664349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2701479,"internal_metadata":4003776345},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000974"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.897282+0000","last_change":"2026-02-19T20:28:49.932388+0000","last_active":"2026-02-19T21:11:24.897282+0000","last_peered":"2026-02-19T21:11:24.897282+0000","last_clean":"2026-02-19T21:11:24.897282+0000","last_became_active":"2026-02-19T20:28:47.929308+0000","last_became_peered":"2026-02-19T20:28:47.929308+0000","last_unstale":"2026-02-19T21:11:24.897282+0000","last_undegraded":"2026-02-19T21:11:24.897282+0000","last_fullsized":"2026-02-19T21:11:24.897282+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:51:59.644924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086370499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.899360+0000","last_change":"2026-02-19T20:28:49.932057+0000","last_active":"2026-02-19T21:11:24.899360+0000","last_peered":"2026-02-19T21:11:24.899360+0000","last_clean":"2026-02-19T21:11:24.899360+0000","last_became_active":"2026-02-19T20:28:47.929421+0000","last_became_peered":"2026-02-19T20:28:47.929421+0000","last_unstale":"2026-02-19T21:11:24.899360+0000","last_undegraded":"2026-02-19T21:11:24.899360+0000","last_fullsized":"2026-02-19T21:11:24.899360+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:53:36.261502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055284699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.899354+0000","last_change":"2026-02-19T20:28:49.932501+0000","last_active":"2026-02-19T21:11:24.899354+0000","last_peered":"2026-02-19T21:11:24.899354+0000","last_clean":"2026-02-19T21:11:24.899354+0000","last_became_active":"2026-02-19T20:28:47.929096+0000","last_became_peered":"2026-02-19T20:28:47.929096+0000","last_unstale":"2026-02-19T21:11:24.899354+0000","last_undegraded":"2026-02-19T21:11:24.899354+0000","last_fullsized":"2026-02-19T21:11:24.899354+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:35:27.823970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001005578,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":697,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.895037+0000","last_change":"2026-02-19T20:28:49.932359+0000","last_active":"2026-02-19T21:11:24.895037+0000","last_peered":"2026-02-19T21:11:24.895037+0000","last_clean":"2026-02-19T21:11:24.895037+0000","last_became_active":"2026-02-19T20:28:47.929286+0000","last_became_peered":"2026-02-19T20:28:47.929286+0000","last_unstale":"2026-02-19T21:11:24.895037+0000","last_undegraded":"2026-02-19T21:11:24.895037+0000","last_fullsized":"2026-02-19T21:11:24.895037+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:15:05.977395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084824599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":700,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.868750+0000","last_change":"2026-02-19T20:28:49.930433+0000","last_active":"2026-02-19T21:11:24.868750+0000","last_peered":"2026-02-19T21:11:24.868750+0000","last_clean":"2026-02-19T21:11:24.868750+0000","last_became_active":"2026-02-19T20:28:47.929119+0000","last_became_peered":"2026-02-19T20:28:47.929119+0000","last_unstale":"2026-02-19T21:11:24.868750+0000","last_undegraded":"2026-02-19T21:11:24.868750+0000","last_fullsized":"2026-02-19T21:11:24.868750+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T04:02:22.704407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092100199999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":698,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.894576+0000","last_change":"2026-02-19T20:28:50.333526+0000","last_active":"2026-02-19T21:11:24.894576+0000","last_peered":"2026-02-19T21:11:24.894576+0000","last_clean":"2026-02-19T21:11:24.894576+0000","last_became_active":"2026-02-19T20:28:47.929462+0000","last_became_peered":"2026-02-19T20:28:47.929462+0000","last_unstale":"2026-02-19T21:11:24.894576+0000","last_undegraded":"2026-02-19T21:11:24.894576+0000","last_fullsized":"2026-02-19T21:11:24.894576+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:14:47.461186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024374299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":699,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.894122+0000","last_change":"2026-02-19T20:28:50.333516+0000","last_active":"2026-02-19T21:11:24.894122+0000","last_peered":"2026-02-19T21:11:24.894122+0000","last_clean":"2026-02-19T21:11:24.894122+0000","last_became_active":"2026-02-19T20:28:47.929487+0000","last_became_peered":"2026-02-19T20:28:47.929487+0000","last_unstale":"2026-02-19T21:11:24.894122+0000","last_undegraded":"2026-02-19T21:11:24.894122+0000","last_fullsized":"2026-02-19T21:11:24.894122+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T03:04:05.217623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024657800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":698,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.897258+0000","last_change":"2026-02-19T20:28:49.932091+0000","last_active":"2026-02-19T21:11:24.897258+0000","last_peered":"2026-02-19T21:11:24.897258+0000","last_clean":"2026-02-19T21:11:24.897258+0000","last_became_active":"2026-02-19T20:28:47.928693+0000","last_became_peered":"2026-02-19T20:28:47.928693+0000","last_unstale":"2026-02-19T21:11:24.897258+0000","last_undegraded":"2026-02-19T21:11:24.897258+0000","last_fullsized":"2026-02-19T21:11:24.897258+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:46.917072+0000","last_clean_scrub_stamp":"2026-02-19T20:28:46.917072+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:23:54.434881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060120699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":787,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:11:24.895975+0000","last_change":"2026-02-19T20:28:45.921802+0000","last_active":"2026-02-19T21:11:24.895975+0000","last_peered":"2026-02-19T21:11:24.895975+0000","last_clean":"2026-02-19T21:11:24.895975+0000","last_became_active":"2026-02-19T20:28:45.920826+0000","last_became_peered":"2026-02-19T20:28:45.920826+0000","last_unstale":"2026-02-19T21:11:24.895975+0000","last_undegraded":"2026-02-19T21:11:24.895975+0000","last_fullsized":"2026-02-19T21:11:24.895975+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T20:28:44.914354+0000","last_clean_scrub_stamp":"2026-02-19T20:28:44.914354+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T20:32:41.646846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738891,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":511256,"kb_used_data":2760,"kb_used_omap":1385,"kb_used_meta":507094,"kb_avail":93860584,"statfs":{"total":96636764160,"available":96113238016,"internally_reserved":0,"allocated":2826240,"data_stored":1494525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1418428,"internal_metadata":519265092},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738894,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1413856,"kb_used_data":3344,"kb_used_omap":577,"kb_used_meta":1409918,"kb_avail":92957984,"statfs":{"total":96636764160,"available":95188975616,"internally_reserved":0,"allocated":3424256,"data_stored":2084912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":591262,"internal_metadata":1443756642},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738893,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1996960,"kb_used_data":3344,"kb_used_omap":675,"kb_used_meta":1992924,"kb_avail":92374880,"statfs":{"total":96636764160,"available":94591877120,"internally_reserved":0,"allocated":3424256,"data_stored":2084912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":691789,"internal_metadata":2040754611},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T21:12:12.655 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-19T20:28:46.917072+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=28, tm_sec=46, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=12, tm_sec=10, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:12:12.656 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-19T20:28:46.917072+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=28, tm_sec=46, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=12, tm_sec=10, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:12:12.656 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-19T20:28:46.917072+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=28, tm_sec=46, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=12, tm_sec=10, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:12:12.656 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-19T20:28:46.917072+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=28, tm_sec=46, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=12, tm_sec=10, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:12:12.656 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-19T20:28:46.917072+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=28, tm_sec=46, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=12, tm_sec=10, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:12:12.656 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-19T20:28:46.917072+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=28, tm_sec=46, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=12, tm_sec=10, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:12:12.656 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-19T20:28:46.917072+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=28, tm_sec=46, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=12, tm_sec=10, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:12:12.657 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-19T20:28:46.917072+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=28, tm_sec=46, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=12, tm_sec=10, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:12:12.657 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-19T20:28:44.914354+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=20, tm_min=28, tm_sec=44, tm_wday=3, tm_yday=50, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=19, tm_hour=21, tm_min=12, tm_sec=10, tm_wday=3, tm_yday=50, tm_isdst=0) 2026-02-19T21:12:12.657 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-19T21:12:32.657 DEBUG:teuthology.orchestra.run.trial013:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-19T21:12:32.814 INFO:teuthology.orchestra.run.trial013.stdout: 2026-02-19T21:12:32.815 INFO:teuthology.orchestra.run.trial013.stderr:dumped all 2026-02-19T21:12:32.825 INFO:teuthology.orchestra.run.trial013.stdout:{"pg_ready":true,"pg_map":{"version":1655,"stamp":"2026-02-19T21:12:31.293863+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3922072,"kb_used_data":9448,"kb_used_omap":2638,"kb_used_meta":3909937,"kb_avail":279193448,"statfs":{"total":289910292480,"available":285894090752,"internally_reserved":0,"allocated":9674752,"data_stored":5664349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2701479,"internal_metadata":4003776345},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001015"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":705,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:12:17.388673+0000","last_change":"2026-02-19T21:12:17.388673+0000","last_active":"2026-02-19T21:12:17.388673+0000","last_peered":"2026-02-19T21:12:17.388673+0000","last_clean":"2026-02-19T21:12:17.388673+0000","last_became_active":"2026-02-19T20:28:47.929308+0000","last_became_peered":"2026-02-19T20:28:47.929308+0000","last_unstale":"2026-02-19T21:12:17.388673+0000","last_undegraded":"2026-02-19T21:12:17.388673+0000","last_fullsized":"2026-02-19T21:12:17.388673+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:12:17.388582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:12:17.388582+0000","last_clean_scrub_stamp":"2026-02-19T21:12:17.388582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T22:10:21.172162+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0.00086370499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":705,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:12:12.425957+0000","last_change":"2026-02-19T21:12:12.425957+0000","last_active":"2026-02-19T21:12:12.425957+0000","last_peered":"2026-02-19T21:12:12.425957+0000","last_clean":"2026-02-19T21:12:12.425957+0000","last_became_active":"2026-02-19T20:28:47.929421+0000","last_became_peered":"2026-02-19T20:28:47.929421+0000","last_unstale":"2026-02-19T21:12:12.425957+0000","last_undegraded":"2026-02-19T21:12:12.425957+0000","last_fullsized":"2026-02-19T21:12:12.425957+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:12:12.425920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:12:12.425920+0000","last_clean_scrub_stamp":"2026-02-19T21:12:12.425920+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T06:34:40.074027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055284699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":705,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:12:16.368456+0000","last_change":"2026-02-19T21:12:16.368456+0000","last_active":"2026-02-19T21:12:16.368456+0000","last_peered":"2026-02-19T21:12:16.368456+0000","last_clean":"2026-02-19T21:12:16.368456+0000","last_became_active":"2026-02-19T20:28:47.929096+0000","last_became_peered":"2026-02-19T20:28:47.929096+0000","last_unstale":"2026-02-19T21:12:16.368456+0000","last_undegraded":"2026-02-19T21:12:16.368456+0000","last_fullsized":"2026-02-19T21:12:16.368456+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:12:16.368367+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:12:16.368367+0000","last_clean_scrub_stamp":"2026-02-19T21:12:16.368367+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T22:16:40.876517+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0.001005578,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":705,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:12:15.351163+0000","last_change":"2026-02-19T21:12:15.351163+0000","last_active":"2026-02-19T21:12:15.351163+0000","last_peered":"2026-02-19T21:12:15.351163+0000","last_clean":"2026-02-19T21:12:15.351163+0000","last_became_active":"2026-02-19T20:28:47.929286+0000","last_became_peered":"2026-02-19T20:28:47.929286+0000","last_unstale":"2026-02-19T21:12:15.351163+0000","last_undegraded":"2026-02-19T21:12:15.351163+0000","last_fullsized":"2026-02-19T21:12:15.351163+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:12:15.351108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:12:15.351108+0000","last_clean_scrub_stamp":"2026-02-19T21:12:15.351108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T23:59:11.535630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084824599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"343'3","reported_seq":709,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:12:12.500130+0000","last_change":"2026-02-19T21:12:12.500130+0000","last_active":"2026-02-19T21:12:12.500130+0000","last_peered":"2026-02-19T21:12:12.500130+0000","last_clean":"2026-02-19T21:12:12.500130+0000","last_became_active":"2026-02-19T20:28:47.929119+0000","last_became_peered":"2026-02-19T20:28:47.929119+0000","last_unstale":"2026-02-19T21:12:12.500130+0000","last_undegraded":"2026-02-19T21:12:12.500130+0000","last_fullsized":"2026-02-19T21:12:12.500130+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"343'3","last_scrub_stamp":"2026-02-19T21:12:12.500107+0000","last_deep_scrub":"343'3","last_deep_scrub_stamp":"2026-02-19T21:12:12.500107+0000","last_clean_scrub_stamp":"2026-02-19T21:12:12.500107+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T07:22:55.936813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092100199999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":706,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:12:14.136562+0000","last_change":"2026-02-19T21:12:14.136562+0000","last_active":"2026-02-19T21:12:14.136562+0000","last_peered":"2026-02-19T21:12:14.136562+0000","last_clean":"2026-02-19T21:12:14.136562+0000","last_became_active":"2026-02-19T20:28:47.929462+0000","last_became_peered":"2026-02-19T20:28:47.929462+0000","last_unstale":"2026-02-19T21:12:14.136562+0000","last_undegraded":"2026-02-19T21:12:14.136562+0000","last_fullsized":"2026-02-19T21:12:14.136562+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:12:14.135296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:12:14.135296+0000","last_clean_scrub_stamp":"2026-02-19T21:12:14.135296+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T01:55:40.182028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024374299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":707,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:12:13.157102+0000","last_change":"2026-02-19T21:12:13.157102+0000","last_active":"2026-02-19T21:12:13.157102+0000","last_peered":"2026-02-19T21:12:13.157102+0000","last_clean":"2026-02-19T21:12:13.157102+0000","last_became_active":"2026-02-19T20:28:47.929487+0000","last_became_peered":"2026-02-19T20:28:47.929487+0000","last_unstale":"2026-02-19T21:12:13.157102+0000","last_undegraded":"2026-02-19T21:12:13.157102+0000","last_fullsized":"2026-02-19T21:12:13.157102+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-19T21:12:13.156964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-19T21:12:13.156964+0000","last_clean_scrub_stamp":"2026-02-19T21:12:13.156964+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T03:19:10.905624+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0.00024657800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"343'2","reported_seq":707,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:12:14.396320+0000","last_change":"2026-02-19T21:12:14.396320+0000","last_active":"2026-02-19T21:12:14.396320+0000","last_peered":"2026-02-19T21:12:14.396320+0000","last_clean":"2026-02-19T21:12:14.396320+0000","last_became_active":"2026-02-19T20:28:47.928693+0000","last_became_peered":"2026-02-19T20:28:47.928693+0000","last_unstale":"2026-02-19T21:12:14.396320+0000","last_undegraded":"2026-02-19T21:12:14.396320+0000","last_fullsized":"2026-02-19T21:12:14.396320+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"343'2","last_scrub_stamp":"2026-02-19T21:12:14.396270+0000","last_deep_scrub":"343'2","last_deep_scrub_stamp":"2026-02-19T21:12:14.396270+0000","last_clean_scrub_stamp":"2026-02-19T21:12:14.396270+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T03:30:32.837818+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0.00060120699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"343'74","reported_seq":797,"reported_epoch":343,"state":"active+clean","last_fresh":"2026-02-19T21:12:13.393873+0000","last_change":"2026-02-19T21:12:13.393873+0000","last_active":"2026-02-19T21:12:13.393873+0000","last_peered":"2026-02-19T21:12:13.393873+0000","last_clean":"2026-02-19T21:12:13.393873+0000","last_became_active":"2026-02-19T20:28:45.920826+0000","last_became_peered":"2026-02-19T20:28:45.920826+0000","last_unstale":"2026-02-19T21:12:13.393873+0000","last_undegraded":"2026-02-19T21:12:13.393873+0000","last_fullsized":"2026-02-19T21:12:13.393873+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"343'74","last_scrub_stamp":"2026-02-19T21:12:13.393751+0000","last_deep_scrub":"343'74","last_deep_scrub_stamp":"2026-02-19T21:12:13.393751+0000","last_clean_scrub_stamp":"2026-02-19T21:12:13.393751+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-20T21:28:30.525547+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738895,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":511256,"kb_used_data":2760,"kb_used_omap":1385,"kb_used_meta":507094,"kb_avail":93860584,"statfs":{"total":96636764160,"available":96113238016,"internally_reserved":0,"allocated":2826240,"data_stored":1494525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1418428,"internal_metadata":519265092},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738898,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1413856,"kb_used_data":3344,"kb_used_omap":577,"kb_used_meta":1409918,"kb_avail":92957984,"statfs":{"total":96636764160,"available":95188975616,"internally_reserved":0,"allocated":3424256,"data_stored":2084912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":591262,"internal_metadata":1443756642},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738897,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1996960,"kb_used_data":3344,"kb_used_omap":675,"kb_used_meta":1992924,"kb_avail":92374880,"statfs":{"total":96636764160,"available":94591877120,"internally_reserved":0,"allocated":3424256,"data_stored":2084912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":691789,"internal_metadata":2040754611},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-19T21:12:32.827 DEBUG:teuthology.orchestra.run.trial013:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-19T21:12:32.994 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-19T21:12:32.994 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-19T21:12:32.994 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-19T21:12:32.994 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:12:35.787 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.0 is failed for ~0s 2026-02-19T21:12:39.096 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-19T21:12:39.096 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-19T21:12:39.097 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:12:41.091 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 is failed for ~0s 2026-02-19T21:12:41.092 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.0 has been restored 2026-02-19T21:12:45.198 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-19T21:12:45.198 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-19T21:12:45.198 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:12:46.294 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 is failed for ~0s 2026-02-19T21:12:46.294 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 has been restored 2026-02-19T21:12:51.299 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-19T21:12:51.299 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-19T21:12:51.300 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-19T21:12:51.300 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:12:51.331 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-19T21:12:51.331 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-19T21:12:51.331 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-19T21:12:51.331 INFO:teuthology.orchestra.run:waiting for 300 2026-02-19T21:12:51.389 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 is failed for ~5s 2026-02-19T21:12:51.390 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.mon.a is failed for ~0s 2026-02-19T21:12:51.390 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-19T21:12:51.390 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-19T21:12:51.390 DEBUG:teuthology.orchestra.run.trial013:> sudo grep -E '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | grep -E -v '\(MDS_ALL_DOWN\)' | grep -E -v '\(MDS_UP_LESS_THAN_MAX\)' | grep -E -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-19T21:12:51.447 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial013.front.sepia.ceph.com 2026-02-19T21:12:51.447 DEBUG:teuthology.orchestra.run.trial013:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-19T21:12:52.311 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial013.front.sepia.ceph.com 2026-02-19T21:12:52.312 DEBUG:teuthology.orchestra.run.trial013:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-19T21:12:53.064 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial013.front.sepia.ceph.com 2026-02-19T21:12:53.064 DEBUG:teuthology.orchestra.run.trial013:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-19T21:12:53.857 INFO:tasks.ceph:Archiving mon data... 2026-02-19T21:12:53.857 DEBUG:teuthology.misc:Transferring archived files from trial013:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645/data/cluster1.mon.a.tgz 2026-02-19T21:12:53.858 DEBUG:teuthology.orchestra.run.trial013:> mktemp 2026-02-19T21:12:53.863 INFO:teuthology.orchestra.run.trial013.stdout:/tmp/tmp.7greDBL8iR 2026-02-19T21:12:53.863 DEBUG:teuthology.orchestra.run.trial013:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.7greDBL8iR 2026-02-19T21:12:53.970 DEBUG:teuthology.orchestra.run.trial013:> sudo chmod 0666 /tmp/tmp.7greDBL8iR 2026-02-19T21:12:54.027 DEBUG:teuthology.orchestra.remote:trial013:/tmp/tmp.7greDBL8iR is 419KB 2026-02-19T21:12:54.086 DEBUG:teuthology.orchestra.run.trial013:> rm -fr /tmp/tmp.7greDBL8iR 2026-02-19T21:12:54.089 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-19T21:12:54.090 DEBUG:teuthology.orchestra.run.trial013:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-19T21:12:54.138 DEBUG:teuthology.orchestra.run.trial151:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-19T21:12:54.196 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-19T21:12:54.200 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-19T21:12:54.200 INFO:tasks.ceph:Archiving crash dumps... 2026-02-19T21:12:54.202 DEBUG:teuthology.misc:Transferring archived files from trial013:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645/remote/trial013/crash 2026-02-19T21:12:54.204 DEBUG:teuthology.orchestra.run.trial013:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-19T21:12:54.250 DEBUG:teuthology.misc:Transferring archived files from trial151:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645/remote/trial151/crash 2026-02-19T21:12:54.251 DEBUG:teuthology.orchestra.run.trial151:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-19T21:12:54.262 INFO:tasks.ceph:Compressing logs... 2026-02-19T21:12:54.262 DEBUG:teuthology.orchestra.run.trial013:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-19T21:12:54.299 DEBUG:teuthology.orchestra.run.trial151:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-19T21:12:54.331 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18807.log 2026-02-19T21:12:54.331 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18708.log 2026-02-19T21:12:54.331 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18807.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-19T21:12:54.331 INFO:teuthology.orchestra.run.trial013.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18807.log.gz 2026-02-19T21:12:54.331 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18708.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20020.log 2026-02-19T21:12:54.331 INFO:teuthology.orchestra.run.trial013.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.admin.18708.log.gz 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19994.log 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-mon.a.log: /var/log/ceph/cluster1-client.admin.20020.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19041.log 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20020.log.gz 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.19994.log: gzip -5 0.0% --verbose -- -- replaced with /var/log/ceph/cluster1-client.admin.19994.log.gz /var/log/ceph/cluster1-client.admin.19864.log 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr: 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.19041.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19041.log.gz 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20098.log 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.19864.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19864.log.gz 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-19T21:12:54.332 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-osd.1.log: /var/log/ceph/cluster1-client.admin.20098.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20098.log.gz 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19015.log 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18755.log 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15785.log 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.19015.log: /var/log/ceph/cluster1-client.admin.18755.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19015.log.gz 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18755.log.gz 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15985.log 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.15785.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15785.log.gz 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.15985.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15985.log.gz 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18658.log 2026-02-19T21:12:54.333 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18658.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20124.log 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18658.log.gz 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16057.log 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr: 93.2% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.20124.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19916.log 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20124.log.gz 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.16057.log: 29.2% -- replaced with /var/log/ceph/cluster1-client.admin.16057.log.gz 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18839.log 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18683.log 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.19916.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19916.log.gz 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18839.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18839.log.gz 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18917.log 2026-02-19T21:12:54.334 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18683.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18683.log.gz 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18729.log 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18919.log 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18917.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18917.log.gz 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18729.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18729.log.gz 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20072.log 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18919.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18919.log.gz 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18989.log 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18781.log 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.20072.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20072.log.gz 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr: 87.8% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-19T21:12:54.335 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18989.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18989.log.gz 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19942.log 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18781.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15857.log 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18781.log.gz 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.12166.log 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.19942.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19942.log.gz 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.15857.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15857.log.gz 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/ceph.tmp-client.admin.12166.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.12166.log.gz 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20174.log 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15950.log 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18841.log 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.20174.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20174.log.gz 2026-02-19T21:12:54.336 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.15950.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15902.log 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15950.log.gz 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18841.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20150.log 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18841.log.gz 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19890.log 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.15902.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15902.log.gz 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.20150.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20150.log.gz 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20046.log 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.19890.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19890.log.gz 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15926.log 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19968.log 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.20046.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20046.log.gz 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-19T21:12:54.337 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.15926.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15926.log.gz 2026-02-19T21:12:54.338 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.19968.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18918.log 2026-02-19T21:12:54.338 INFO:teuthology.orchestra.run.trial013.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19968.log.gz 2026-02-19T21:12:54.338 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18840.log 2026-02-19T21:12:54.338 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18918.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18918.log.gz 2026-02-19T21:12:54.338 INFO:teuthology.orchestra.run.trial013.stderr:/var/log/ceph/cluster1-client.admin.18840.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18840.log.gz 2026-02-19T21:12:54.355 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115254.log 2026-02-19T21:12:54.355 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98618.log 2026-02-19T21:12:54.355 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113370.log 2026-02-19T21:12:54.355 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115254.log.gz 2026-02-19T21:12:54.355 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125292.log 2026-02-19T21:12:54.355 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98618.log: /var/log/ceph/cluster2-client.mirror.113370.log: 88.3%gzip -5 -- replaced with /var/log/ceph/cluster2-client.mirror.98618.log.gz --verbose -- 2026-02-19T21:12:54.355 INFO:teuthology.orchestra.run.trial151.stderr: /var/log/ceph/cluster2-client.mirror.113156.log 2026-02-19T21:12:54.355 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113370.log.gz 2026-02-19T21:12:54.355 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125292.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86198.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125292.log.gz 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113156.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121986.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86198.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37636.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.86198.log.gz 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121986.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22251.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102421.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37636.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.113156.log.gz 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22251.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101842.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127544.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.22251.log.gz 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102421.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121986.log.gz 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102421.log.gz 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.101842.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116316.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.127544.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110007.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.101842.log.gz 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127544.log.gz 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106860.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116316.log.gz 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110007.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43916.log 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110007.log.gz 2026-02-19T21:12:54.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106860.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105120.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106860.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43916.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67065.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21507.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105120.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43916.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81252.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67065.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105120.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21507.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92335.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67065.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81252.log: 97.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37636.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71411.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21507.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81252.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92335.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92335.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102817.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71411.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34450.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59866.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102817.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71411.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34450.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102817.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72818.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.120762.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59866.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34450.log.gz 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72818.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65513.log 2026-02-19T21:12:54.357 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59866.log.gz 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.1.120762.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126183.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53449.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65513.log: /var/log/ceph/cluster1-client.mirror.126183.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126183.log.gz 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76729.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65513.log.gz 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124006.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53449.log: /var/log/ceph/cluster2-client.mirror.76729.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53449.log.gz 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66545.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18663.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124006.log.gz 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66545.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87933.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66545.log.gz 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18663.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94018.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr: 80.6% -- replaced with /var/log/ceph/cluster2-client.admin.18663.log.gz 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.1.120762.log.gz 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87933.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124163.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94018.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74298.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68485.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87933.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.94018.log.gz 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124163.log: /var/log/ceph/cluster2-client.mirror.74298.log: gzip -5 87.0% --verbose -- -- replaced with /var/log/ceph/cluster1-client.mirror.124163.log.gz /var/log/ceph/cluster1-client.mirror.124456.log 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:54.358 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74298.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.72818.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68485.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121915.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76729.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124456.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68485.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.124456.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.30641.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71473.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.1.30641.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119997.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123083.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71473.log: /var/log/ceph/cluster2-client.mirror.119997.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121915.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71473.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46921.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.119997.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124906.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123083.log: /var/log/ceph/cluster2-client.mirror.46921.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123083.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44856.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46921.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124906.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39386.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124906.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.44856.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45922.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.44856.log.gz 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39386.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23845.log 2026-02-19T21:12:54.359 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123556.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45922.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39386.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.23845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43175.log 88.8% 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.45922.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123556.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.23845.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36390.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123556.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79403.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120335.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36390.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43175.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79403.log: gzip -5 --verbose -- 84.7% /var/log/ceph/cluster2-client.mirror.111222.log -- replaced with /var/log/ceph/cluster2-client.mirror.79403.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36390.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120335.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128437.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.120335.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119853.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111222.log.gz/var/log/ceph/cluster1-client.admin.128437.log: 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124636.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.128437.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119853.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105849.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104278.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124636.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105849.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85466.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105849.log.gz 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103673.log 2026-02-19T21:12:54.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104278.log: 90.7%/var/log/ceph/cluster2-client.mirror.85466.log: -- replaced with /var/log/ceph/cluster2-client.mirror.119853.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104278.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113242.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85466.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120242.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 88.3% --verbose -- -- replaced with /var/log/ceph/cluster2-client.mirror.103673.log.gz /var/log/ceph/cluster1-client.mirror.128201.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113242.log: /var/log/ceph/cluster2-client.mirror.120242.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126227.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113242.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.120242.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.128201.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51505.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128201.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126227.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102623.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126227.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51505.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71376.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51505.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102623.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107131.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102623.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52770.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71376.log: /var/log/ceph/cluster2-client.mirror.107131.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94855.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107131.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71376.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52770.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91282.log 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52770.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94855.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.94855.log.gz 2026-02-19T21:12:54.361 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102995.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107682.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.91282.log: /var/log/ceph/cluster2-client.mirror.102995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49509.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102995.log.gz 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.91282.log.gz 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107682.log: gzip 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107682.log.gz -5 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.101440.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49509.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19326.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49509.log.gz 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48958.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101440.log: /var/log/ceph/cluster2-client.admin.19326.log: gzip 0.0% -5 -- replaced with /var/log/ceph/cluster2-client.admin.19326.log.gz 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.58931.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101440.log.gz 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48958.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106895.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48958.log.gz 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83493.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58931.log: /var/log/ceph/cluster2-client.mirror.106895.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.106895.log.gz 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58931.log.gz 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44966.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83493.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57961.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76514.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44966.log: /var/log/ceph/cluster2-client.mirror.57961.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.126472.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57961.log.gz 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51796.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76514.log: /var/log/ceph/cluster2-client.mirror.3.126472.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61069.log 2026-02-19T21:12:54.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51796.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51796.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40748.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61069.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24249.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40748.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61069.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44048.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24249.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77333.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.3.126472.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40748.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37937.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24249.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77333.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39157.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76514.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37937.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44048.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77034.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77333.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.1.30641.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39157.log: gzip 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37937.log.gz -5 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.120384.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44966.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77034.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39157.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118790.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120384.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45763.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118790.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65387.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120384.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56118.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45763.log: /var/log/ceph/cluster1-client.admin.65387.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45763.log.gz 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121331.log 2026-02-19T21:12:54.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56118.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.65387.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128057.log 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56118.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.118790.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121331.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99673.log 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.121331.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.128057.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26771.log 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77034.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53627.log 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99673.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128057.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26771.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110039.log 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53627.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26771.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76292.log 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53627.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.110039.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34547.log 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76292.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126128.log 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.110039.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76292.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34547.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34062.log 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126128.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34547.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126586.log 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34062.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126128.log.gz 2026-02-19T21:12:54.364 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36258.log 2026-02-19T21:12:54.369 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126586.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69527.log 2026-02-19T21:12:54.369 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34062.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.126586.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36258.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60095.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69527.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37571.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36258.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60095.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69527.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60095.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59222.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.37571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125809.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59222.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59222.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31996.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84103.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.125809.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31996.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124523.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31996.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.84103.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26383.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124523.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81059.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124523.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26383.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37719.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81059.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26383.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128658.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81059.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.37719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105494.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128658.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.37719.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41136.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128658.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35191.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105494.log: /var/log/ceph/cluster2-client.mirror.41136.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82881.log 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.37571.log.gz 2026-02-19T21:12:54.370 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41136.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35191.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123533.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.35191.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125145.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82881.log: /var/log/ceph/cluster1-client.mirror.123533.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.105494.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75464.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr: 85.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123533.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125145.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.125145.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45062.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75464.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.82729.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75464.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.45062.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124388.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.45062.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.2.82729.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64755.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124388.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25316.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.124388.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64755.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56859.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64755.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49447.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56859.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44924.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49447.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56859.log.gz 88.2% 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.25316.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49447.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120110.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44924.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50479.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54566.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120110.log: /var/log/ceph/cluster2-client.mirror.50479.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97682.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54566.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50479.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106093.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54566.log.gz 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:gzip/var/log/ceph/cluster1-client.mirror.97682.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37459.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101127.log 2026-02-19T21:12:54.371 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124096.log 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.120110.log.gz 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37459.log: /var/log/ceph/cluster2-client.mirror.101127.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.97682.log.gz 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr:gzip 88.1% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.101127.log.gz 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.23508.log 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37459.log.gz 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100577.log 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124096.log: /var/log/ceph/cluster2-client.mirror.23508.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124096.log.gz 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108469.log 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100577.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92921.log 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44924.log.gz 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104888.log 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108469.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100577.log.gz 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92921.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97570.log 2026-02-19T21:12:54.372 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92921.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104888.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.69715.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104888.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124051.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97570.log: /var/log/ceph/cluster1-client.mirror.0.69715.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44436.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97570.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124051.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.106093.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124051.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64526.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23309.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44436.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103708.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64526.log: /var/log/ceph/cluster2-client.mirror.23309.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44436.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99311.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64526.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.23309.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88471.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103708.log: /var/log/ceph/cluster2-client.mirror.99311.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47309.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99311.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103708.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88471.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.88471.log.gz 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85938.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107647.log 2026-02-19T21:12:54.374 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47309.log: /var/log/ceph/cluster2-client.mirror.85938.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118476.log 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85938.log.gz 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47309.log.gz 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107647.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107647.log.gz 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99222.log 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118476.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113643.log 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118476.log.gz 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47274.log 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.99222.log.gz 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94500.log 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113643.log.gz 2026-02-19T21:12:54.375 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47274.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19526.log 2026-02-19T21:12:54.376 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47274.log.gz 2026-02-19T21:12:54.376 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94500.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98812.log 2026-02-19T21:12:54.376 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94500.log.gz 2026-02-19T21:12:54.376 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19526.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117964.log 2026-02-19T21:12:54.376 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.98812.log: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.19526.log.gz 2026-02-19T21:12:54.376 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57670.log 2026-02-19T21:12:54.377 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69199.log 2026-02-19T21:12:54.377 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117964.log: /var/log/ceph/cluster2-client.mirror.57670.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.98812.log.gz 2026-02-19T21:12:54.377 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.117964.log.gz 2026-02-19T21:12:54.377 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77899.log 2026-02-19T21:12:54.377 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57670.log.gz 2026-02-19T21:12:54.378 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69199.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123668.log 2026-02-19T21:12:54.378 INFO:teuthology.orchestra.run.trial151.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69199.log.gz 2026-02-19T21:12:54.378 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77899.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56762.log 2026-02-19T21:12:54.378 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81326.log 2026-02-19T21:12:54.378 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123668.log: /var/log/ceph/cluster2-client.mirror.56762.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56762.log.gz 2026-02-19T21:12:54.379 INFO:teuthology.orchestra.run.trial151.stderr: 86.9%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.123668.log.gz -5 2026-02-19T21:12:54.379 INFO:teuthology.orchestra.run.trial151.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.105031.log 2026-02-19T21:12:54.379 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116351.log 2026-02-19T21:12:54.379 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81326.log: /var/log/ceph/cluster2-client.mirror.105031.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64658.log 2026-02-19T21:12:54.379 INFO:teuthology.orchestra.run.trial151.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81326.log.gz 2026-02-19T21:12:54.379 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.105031.log.gz 2026-02-19T21:12:54.379 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47891.log 2026-02-19T21:12:54.380 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116351.log: /var/log/ceph/cluster2-client.mirror.64658.log: 88.3%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.116351.log.gz 2026-02-19T21:12:54.380 INFO:teuthology.orchestra.run.trial151.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22344.log 2026-02-19T21:12:54.380 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64658.log.gz 2026-02-19T21:12:54.380 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47891.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90434.log 2026-02-19T21:12:54.380 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47891.log.gz 2026-02-19T21:12:54.380 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22344.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34838.log 2026-02-19T21:12:54.381 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90434.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54119.log 2026-02-19T21:12:54.381 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22344.log.gz 2026-02-19T21:12:54.381 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34838.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90434.log.gz 2026-02-19T21:12:54.381 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120710.log 2026-02-19T21:12:54.382 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34256.log 2026-02-19T21:12:54.382 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.54119.log: /var/log/ceph/cluster2-client.mirror.120710.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34838.log.gz 2026-02-19T21:12:54.382 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58446.log 2026-02-19T21:12:54.382 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.54119.log.gz 2026-02-19T21:12:54.382 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34256.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24505.log 2026-02-19T21:12:54.383 INFO:teuthology.orchestra.run.trial151.stderr: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.120710.log.gz 2026-02-19T21:12:54.383 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58446.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106336.log 2026-02-19T21:12:54.383 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34256.log.gz 2026-02-19T21:12:54.383 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24505.log: gzip -5 89.0% --verbose -- -- replaced with /var/log/ceph/cluster2-client.mirror.58446.log.gz /var/log/ceph/cluster2-client.mirror.41101.log 2026-02-19T21:12:54.383 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:54.383 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106336.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.106336.log.gz 2026-02-19T21:12:54.383 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65006.log 2026-02-19T21:12:54.384 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose 88.1% -- /var/log/ceph/cluster2-client.mirror.70115.log -- replaced with /var/log/ceph/cluster2-client.mirror.24505.log.gz 2026-02-19T21:12:54.384 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:54.384 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41101.log: /var/log/ceph/cluster2-client.mirror.65006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88568.log 2026-02-19T21:12:54.384 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41101.log.gz 2026-02-19T21:12:54.385 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70115.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59187.log 2026-02-19T21:12:54.385 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70115.log.gz 2026-02-19T21:12:54.385 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88568.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114617.log 2026-02-19T21:12:54.385 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59187.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88568.log.gz 2026-02-19T21:12:54.385 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86796.log 2026-02-19T21:12:54.385 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59187.log.gz 2026-02-19T21:12:54.386 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.114617.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128243.log 2026-02-19T21:12:54.386 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.114617.log.gz 2026-02-19T21:12:54.386 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.86796.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121215.log 2026-02-19T21:12:54.386 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.86796.log.gz 2026-02-19T21:12:54.387 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115188.log 2026-02-19T21:12:54.387 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.128243.log: /var/log/ceph/cluster2-client.mirror.121215.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93302.log 2026-02-19T21:12:54.387 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128243.log.gz 2026-02-19T21:12:54.387 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.115188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21821.log 2026-02-19T21:12:54.388 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93302.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46151.log 2026-02-19T21:12:54.388 INFO:teuthology.orchestra.run.trial151.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65006.log.gz 2026-02-19T21:12:54.388 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93302.log.gz 2026-02-19T21:12:54.388 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50611.log 2026-02-19T21:12:54.388 INFO:teuthology.orchestra.run.trial151.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121215.log.gz 2026-02-19T21:12:54.388 INFO:teuthology.orchestra.run.trial151.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.115188.log.gz 2026-02-19T21:12:54.389 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46151.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39060.log 2026-02-19T21:12:54.389 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.21821.log.gz 2026-02-19T21:12:54.389 INFO:teuthology.orchestra.run.trial151.stderr: 88.7%/var/log/ceph/cluster2-client.mirror.50611.log: -- replaced with /var/log/ceph/cluster2-client.mirror.46151.log.gzgzip 2026-02-19T21:12:54.389 INFO:teuthology.orchestra.run.trial151.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124965.log 2026-02-19T21:12:54.389 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39060.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50611.log.gz 2026-02-19T21:12:54.389 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52576.log 2026-02-19T21:12:54.390 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.124965.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.124965.log.gz 2026-02-19T21:12:54.390 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39060.log.gz 2026-02-19T21:12:54.390 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63688.log 2026-02-19T21:12:54.390 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52576.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33674.log 2026-02-19T21:12:54.390 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25960.log 2026-02-19T21:12:54.390 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52576.log.gz 2026-02-19T21:12:54.391 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63688.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83552.log 2026-02-19T21:12:54.391 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33674.log: /var/log/ceph/cluster2-client.mirror.25960.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63688.log.gz 2026-02-19T21:12:54.391 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72055.log 2026-02-19T21:12:54.391 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33674.log.gz 2026-02-19T21:12:54.391 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83552.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83552.log.gz 2026-02-19T21:12:54.391 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103480.log 2026-02-19T21:12:54.392 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.25960.log.gz 2026-02-19T21:12:54.392 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72055.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35808.log 2026-02-19T21:12:54.392 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103480.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72055.log.gz 2026-02-19T21:12:54.392 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103480.log.gz 2026-02-19T21:12:54.392 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113787.log 2026-02-19T21:12:54.393 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35808.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.19478.log 2026-02-19T21:12:54.393 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.113787.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29165.log 2026-02-19T21:12:54.393 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35808.log.gz 2026-02-19T21:12:54.393 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.2.19478.log: gzip -5 --verbose -- 88.3% /var/log/ceph/cluster2-client.mirror.37212.log -- replaced with /var/log/ceph/cluster1-client.mirror.113787.log.gz 2026-02-19T21:12:54.393 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:54.394 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121773.log 2026-02-19T21:12:54.394 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62233.log 2026-02-19T21:12:54.394 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37212.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29165.log.gz 2026-02-19T21:12:54.394 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121773.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85373.log 2026-02-19T21:12:54.395 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37212.log.gz 2026-02-19T21:12:54.395 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15877.log 2026-02-19T21:12:54.395 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85373.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127075.log 2026-02-19T21:12:54.395 INFO:teuthology.orchestra.run.trial151.stderr: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.121773.log.gz 2026-02-19T21:12:54.395 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62233.log.gz 2026-02-19T21:12:54.395 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85373.log.gz 2026-02-19T21:12:54.395 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.15877.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76534.log 2026-02-19T21:12:54.395 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15877.log.gz 2026-02-19T21:12:54.396 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.127075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117382.log 2026-02-19T21:12:54.396 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76534.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.127075.log.gz 2026-02-19T21:12:54.396 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62621.log 2026-02-19T21:12:54.396 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117382.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116813.log 2026-02-19T21:12:54.396 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117382.log.gz 2026-02-19T21:12:54.397 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62621.log: gzip -5 --verbose 88.9% -- -- replaced with /var/log/ceph/cluster2-client.mirror.62621.log.gz 2026-02-19T21:12:54.397 INFO:teuthology.orchestra.run.trial151.stderr: /var/log/ceph/cluster2-client.mirror.29006.log 2026-02-19T21:12:54.397 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127565.log 2026-02-19T21:12:54.397 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116813.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103785.log 2026-02-19T21:12:54.397 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29006.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116813.log.gz 2026-02-19T21:12:54.397 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127565.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92693.log 2026-02-19T21:12:54.397 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127565.log.gz 2026-02-19T21:12:54.398 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29006.log.gz 2026-02-19T21:12:54.398 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28971.log 2026-02-19T21:12:54.398 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.103785.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-19T21:12:54.398 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.92693.log: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.76534.log.gz 2026-02-19T21:12:54.398 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28971.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124568.log 2026-02-19T21:12:54.398 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.92693.log.gz 2026-02-19T21:12:54.399 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98700.log 2026-02-19T21:12:54.399 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28971.log.gz 2026-02-19T21:12:54.399 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124568.log: 86.9%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.124568.log.gz 2026-02-19T21:12:54.399 INFO:teuthology.orchestra.run.trial151.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72753.log 2026-02-19T21:12:54.399 INFO:teuthology.orchestra.run.trial151.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.103785.log.gz 2026-02-19T21:12:54.399 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98700.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52087.log 2026-02-19T21:12:54.399 INFO:teuthology.orchestra.run.trial151.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.19478.log.gz 2026-02-19T21:12:54.400 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.72753.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98700.log.gz 2026-02-19T21:12:54.400 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77631.log 2026-02-19T21:12:54.400 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36293.log 2026-02-19T21:12:54.400 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52087.log: /var/log/ceph/cluster2-client.mirror.77631.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66095.log 2026-02-19T21:12:54.400 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52087.log.gz 2026-02-19T21:12:54.400 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36293.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80058.log 2026-02-19T21:12:54.400 INFO:teuthology.orchestra.run.trial151.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77631.log.gz 2026-02-19T21:12:54.401 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66095.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36293.log.gz 2026-02-19T21:12:54.401 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67356.log 2026-02-19T21:12:54.401 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66095.log.gz 2026-02-19T21:12:54.401 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110849.log 2026-02-19T21:12:54.401 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80058.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80058.log.gz 2026-02-19T21:12:54.401 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67356.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77522.log 2026-02-19T21:12:54.402 INFO:teuthology.orchestra.run.trial151.stderr: 88.7%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.67356.log.gz -5 --verbose 2026-02-19T21:12:54.402 INFO:teuthology.orchestra.run.trial151.stderr: -- /var/log/ceph/cluster2-client.mirror.104942.log 2026-02-19T21:12:54.402 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110849.log: /var/log/ceph/cluster2-client.mirror.77522.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121473.log 2026-02-19T21:12:54.402 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110849.log.gz 2026-02-19T21:12:54.402 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111299.log 2026-02-19T21:12:54.402 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104942.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77522.log.gz 2026-02-19T21:12:54.402 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121473.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.72753.log.gz 2026-02-19T21:12:54.403 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121473.log.gz 2026-02-19T21:12:54.403 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124546.log 2026-02-19T21:12:54.403 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.104942.log.gz 2026-02-19T21:12:54.403 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94939.log 2026-02-19T21:12:54.403 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111299.log: /var/log/ceph/cluster1-client.mirror.124546.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72918.log 2026-02-19T21:12:54.403 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124546.log.gz 2026-02-19T21:12:54.403 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.111299.log.gz 2026-02-19T21:12:54.404 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.94939.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18761.log 2026-02-19T21:12:54.404 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.94939.log.gz 2026-02-19T21:12:54.404 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.72918.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127950.log 2026-02-19T21:12:54.404 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.72918.log.gz 2026-02-19T21:12:54.404 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18761.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18761.log.gz 2026-02-19T21:12:54.404 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70441.log 2026-02-19T21:12:54.405 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127950.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.23895.log 2026-02-19T21:12:54.405 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113014.log 2026-02-19T21:12:54.405 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70441.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127950.log.gz 2026-02-19T21:12:54.406 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.23895.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.75649.log 2026-02-19T21:12:54.406 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70441.log.gz 2026-02-19T21:12:54.406 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113014.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117245.log 2026-02-19T21:12:54.406 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113014.log.gz 2026-02-19T21:12:54.406 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.2.75649.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102658.log 2026-02-19T21:12:54.406 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53158.log 2026-02-19T21:12:54.407 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117245.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112264.log 2026-02-19T21:12:54.407 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102658.log: /var/log/ceph/cluster2-client.mirror.53158.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32481.log 2026-02-19T21:12:54.407 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102658.log.gz 2026-02-19T21:12:54.407 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53158.log.gz 2026-02-19T21:12:54.407 INFO:teuthology.orchestra.run.trial151.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.23895.log.gz/var/log/ceph/cluster2-client.mirror.112264.log: 2026-02-19T21:12:54.407 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.112264.log.gz 2026-02-19T21:12:54.407 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125251.log 2026-02-19T21:12:54.408 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32481.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.117245.log.gz 2026-02-19T21:12:54.408 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45146.log 2026-02-19T21:12:54.408 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32481.log.gz 2026-02-19T21:12:54.408 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65998.log 2026-02-19T21:12:54.408 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125251.log: /var/log/ceph/cluster2-client.mirror.45146.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125251.log.gz 2026-02-19T21:12:54.408 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119571.log 2026-02-19T21:12:54.409 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45146.log.gz 2026-02-19T21:12:54.409 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120966.log 2026-02-19T21:12:54.409 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119571.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65998.log.gz 2026-02-19T21:12:54.409 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.114121.log 2026-02-19T21:12:54.409 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.119571.log.gz 2026-02-19T21:12:54.409 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120966.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77120.log 2026-02-19T21:12:54.410 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.2.114121.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120966.log.gz 2026-02-19T21:12:54.410 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113099.log 2026-02-19T21:12:54.410 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77120.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34935.log 2026-02-19T21:12:54.410 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52122.log 2026-02-19T21:12:54.411 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34935.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.113099.log.gz 2026-02-19T21:12:54.411 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95460.log 2026-02-19T21:12:54.411 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24117.log 2026-02-19T21:12:54.411 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34935.log.gz 2026-02-19T21:12:54.411 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52122.log.gz 2026-02-19T21:12:54.411 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95460.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80540.log 2026-02-19T21:12:54.411 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95460.log.gz 2026-02-19T21:12:54.412 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24117.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120497.log 2026-02-19T21:12:54.412 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87162.log 2026-02-19T21:12:54.412 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80540.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.24117.log.gz 2026-02-19T21:12:54.412 INFO:teuthology.orchestra.run.trial151.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80540.log.gz 2026-02-19T21:12:54.412 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120497.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44789.log 2026-02-19T21:12:54.413 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87162.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34159.log 2026-02-19T21:12:54.413 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.87162.log.gz 2026-02-19T21:12:54.413 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77120.log.gz 2026-02-19T21:12:54.413 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110474.log 2026-02-19T21:12:54.413 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44789.log: /var/log/ceph/cluster2-client.mirror.34159.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60452.log 2026-02-19T21:12:54.413 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44789.log.gz 2026-02-19T21:12:54.414 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34159.log.gz 2026-02-19T21:12:54.414 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122971.log 2026-02-19T21:12:54.414 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110474.log: /var/log/ceph/cluster2-client.mirror.60452.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.120497.log.gz 2026-02-19T21:12:54.414 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93821.log 2026-02-19T21:12:54.414 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110474.log.gz 2026-02-19T21:12:54.414 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60452.log.gz 2026-02-19T21:12:54.414 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81717.log 2026-02-19T21:12:54.415 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.122971.log.gz 2026-02-19T21:12:54.415 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44498.log 2026-02-19T21:12:54.415 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93821.log.gz 2026-02-19T21:12:54.415 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121603.log 2026-02-19T21:12:54.416 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81717.log: /var/log/ceph/cluster2-client.mirror.44498.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81717.log.gz 2026-02-19T21:12:54.416 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46824.log 2026-02-19T21:12:54.416 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121603.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44498.log.gz 2026-02-19T21:12:54.416 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33383.log 2026-02-19T21:12:54.416 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121603.log.gz 2026-02-19T21:12:54.416 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46824.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57441.log 2026-02-19T21:12:54.416 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46824.log.gz 2026-02-19T21:12:54.417 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33383.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70600.log 2026-02-19T21:12:54.417 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57441.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69669.log 2026-02-19T21:12:54.417 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57441.log.gz 2026-02-19T21:12:54.417 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33383.log.gz 2026-02-19T21:12:54.417 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70600.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72887.log 2026-02-19T21:12:54.417 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69669.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.70600.log.gz 2026-02-19T21:12:54.417 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91134.log 2026-02-19T21:12:54.418 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.72887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114550.log 2026-02-19T21:12:54.419 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.91134.log: gzip -5 --verbose 89.2% -- /var/log/ceph/cluster1-client.mirror.127131.log 2026-02-19T21:12:54.419 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.72887.log.gz 2026-02-19T21:12:54.419 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.91134.log.gz 2026-02-19T21:12:54.419 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114550.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114121.log 2026-02-19T21:12:54.420 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114550.log.gz 2026-02-19T21:12:54.420 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.127131.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99365.log 2026-02-19T21:12:54.420 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127131.log.gz 2026-02-19T21:12:54.420 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.114121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106996.log 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99365.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81842.log 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99365.log.gz 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106996.log: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.114121.log.gz 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98093.log 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.106996.log.gz 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52281.log 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81842.log: /var/log/ceph/cluster2-client.mirror.98093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77610.log 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81842.log.gz 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98093.log.gz 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52281.log: gzip 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.2.114121.log.gz 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64332.log 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52281.log.gz 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77610.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119620.log 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102017.log 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr: 96.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69669.log.gz 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64332.log.gz 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77610.log.gz 2026-02-19T21:12:54.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119620.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65069.log 2026-02-19T21:12:54.423 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102017.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36064.log 2026-02-19T21:12:54.423 INFO:teuthology.orchestra.run.trial151.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119620.log.gz 2026-02-19T21:12:54.423 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102017.log.gz 2026-02-19T21:12:54.423 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.65069.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39778.log 2026-02-19T21:12:54.423 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36064.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124298.log 2026-02-19T21:12:54.423 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39778.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45860.log 2026-02-19T21:12:54.423 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36064.log.gz 2026-02-19T21:12:54.423 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124298.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54628.log 2026-02-19T21:12:54.423 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39778.log.gz 2026-02-19T21:12:54.424 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124298.log.gz 2026-02-19T21:12:54.424 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45860.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71861.log 2026-02-19T21:12:54.424 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45860.log.gz 2026-02-19T21:12:54.424 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54628.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95495.log 2026-02-19T21:12:54.424 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71861.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54628.log.gz 2026-02-19T21:12:54.424 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57864.log 2026-02-19T21:12:54.424 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71861.log.gz 2026-02-19T21:12:54.424 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95495.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76663.log 2026-02-19T21:12:54.425 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65069.log.gz 2026-02-19T21:12:54.425 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95495.log.gz 2026-02-19T21:12:54.425 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57864.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29359.log 2026-02-19T21:12:54.425 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76663.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57864.log.gz 2026-02-19T21:12:54.425 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110970.log 2026-02-19T21:12:54.425 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121694.log 2026-02-19T21:12:54.426 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.110970.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29359.log.gz 2026-02-19T21:12:54.426 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48729.log 2026-02-19T21:12:54.426 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42634.log 2026-02-19T21:12:54.426 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121694.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.110970.log.gz 2026-02-19T21:12:54.427 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48729.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46442.log 2026-02-19T21:12:54.427 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48729.log.gz 2026-02-19T21:12:54.427 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42634.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113566.log 2026-02-19T21:12:54.427 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46442.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44401.log 2026-02-19T21:12:54.427 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42634.log.gz 2026-02-19T21:12:54.427 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46442.log.gz 2026-02-19T21:12:54.428 INFO:teuthology.orchestra.run.trial151.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121694.log.gz 2026-02-19T21:12:54.428 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113566.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67030.log 2026-02-19T21:12:54.428 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44401.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113566.log.gz 2026-02-19T21:12:54.428 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26833.log 2026-02-19T21:12:54.428 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67030.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109004.log 2026-02-19T21:12:54.428 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67030.log.gz 2026-02-19T21:12:54.428 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44401.log.gz 2026-02-19T21:12:54.428 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26833.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99750.log 2026-02-19T21:12:54.429 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97092.log 2026-02-19T21:12:54.429 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109004.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.76663.log.gz 2026-02-19T21:12:54.429 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26833.log.gz 2026-02-19T21:12:54.429 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.99750.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46983.log 2026-02-19T21:12:54.429 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97092.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122422.log 2026-02-19T21:12:54.430 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97092.log.gz 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46983.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.109004.log.gz 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36099.log 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46983.log.gz 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122422.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28517.log 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.99750.log.gz 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36099.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.122422.log.gz 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103923.log 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28517.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36099.log.gz 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95333.log 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103923.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103923.log.gz 2026-02-19T21:12:54.431 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61845.log 2026-02-19T21:12:54.432 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28517.log.gz 2026-02-19T21:12:54.432 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95333.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24637.log 2026-02-19T21:12:54.432 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61845.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95333.log.gz 2026-02-19T21:12:54.432 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97907.log 2026-02-19T21:12:54.432 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61845.log.gz 2026-02-19T21:12:54.432 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24637.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43140.log 2026-02-19T21:12:54.433 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24637.log.gz 2026-02-19T21:12:54.433 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121516.log 2026-02-19T21:12:54.433 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97907.log.gz 2026-02-19T21:12:54.433 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34124.log 2026-02-19T21:12:54.433 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99963.log 2026-02-19T21:12:54.434 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121516.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43140.log.gz 2026-02-19T21:12:54.434 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34124.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55536.log 2026-02-19T21:12:54.434 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99963.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.121516.log.gz 2026-02-19T21:12:54.434 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34124.log.gz 2026-02-19T21:12:54.434 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19737.log 2026-02-19T21:12:54.434 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99963.log.gz 2026-02-19T21:12:54.434 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55536.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40616.log 2026-02-19T21:12:54.435 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.19737.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78802.log 2026-02-19T21:12:54.435 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55536.log.gz 2026-02-19T21:12:54.435 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40616.log: 80.3% -- replaced with /var/log/ceph/cluster1-client.admin.19737.log.gz 2026-02-19T21:12:54.435 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118255.log 2026-02-19T21:12:54.435 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78802.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87340.log 2026-02-19T21:12:54.436 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40616.log.gz 2026-02-19T21:12:54.436 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118255.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24020.log 2026-02-19T21:12:54.436 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118255.log.gz 2026-02-19T21:12:54.436 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87340.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113854.log 2026-02-19T21:12:54.436 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.87340.log.gz 2026-02-19T21:12:54.436 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53662.log 2026-02-19T21:12:54.437 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24020.log: /var/log/ceph/cluster2-client.mirror.113854.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125230.log 2026-02-19T21:12:54.437 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53662.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.24020.log.gz 2026-02-19T21:12:54.437 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94139.log 2026-02-19T21:12:54.437 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53662.log.gz 2026-02-19T21:12:54.437 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125230.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106701.log 2026-02-19T21:12:54.438 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94139.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99705.log 2026-02-19T21:12:54.438 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.94139.log.gz 2026-02-19T21:12:54.438 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106701.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123241.log 2026-02-19T21:12:54.438 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106701.log.gz 2026-02-19T21:12:54.438 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.99705.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90639.log 2026-02-19T21:12:54.438 INFO:teuthology.orchestra.run.trial151.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.125230.log.gz 2026-02-19T21:12:54.439 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123241.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27644.log 2026-02-19T21:12:54.439 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.99705.log.gz 2026-02-19T21:12:54.439 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123241.log.gz 2026-02-19T21:12:54.439 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71667.log 2026-02-19T21:12:54.439 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90639.log.gz 2026-02-19T21:12:54.440 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27644.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72579.log 2026-02-19T21:12:54.440 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100542.log 2026-02-19T21:12:54.440 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27644.log.gz 2026-02-19T21:12:54.440 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71667.log.gz 2026-02-19T21:12:54.440 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72579.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128087.log 2026-02-19T21:12:54.441 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100542.log: 89.2%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.72579.log.gz -5 --verbose 2026-02-19T21:12:54.441 INFO:teuthology.orchestra.run.trial151.stderr: -- /var/log/ceph/cluster1-client.mirror.124411.log 2026-02-19T21:12:54.441 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100542.log.gz 2026-02-19T21:12:54.441 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.128087.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128814.log 2026-02-19T21:12:54.441 INFO:teuthology.orchestra.run.trial151.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.128087.log.gz 2026-02-19T21:12:54.441 INFO:teuthology.orchestra.run.trial151.stderr:gzip/var/log/ceph/cluster1-client.mirror.124411.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66836.log 2026-02-19T21:12:54.442 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128814.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124411.log.gz 2026-02-19T21:12:54.442 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128814.log.gz 2026-02-19T21:12:54.442 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65672.log 2026-02-19T21:12:54.442 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63785.log 2026-02-19T21:12:54.442 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65672.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66836.log.gz 2026-02-19T21:12:54.442 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125610.log 2026-02-19T21:12:54.442 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65672.log.gz 2026-02-19T21:12:54.443 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28226.log 2026-02-19T21:12:54.443 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63785.log: /var/log/ceph/cluster2-client.mirror.125610.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63785.log.gz 2026-02-19T21:12:54.443 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32710.log 2026-02-19T21:12:54.443 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28226.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99772.log 2026-02-19T21:12:54.444 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32710.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92111.log 2026-02-19T21:12:54.444 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28226.log.gz 2026-02-19T21:12:54.444 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99772.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74589.log 2026-02-19T21:12:54.444 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32710.log.gz 2026-02-19T21:12:54.444 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92111.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125562.log 2026-02-19T21:12:54.444 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92111.log.gz 2026-02-19T21:12:54.445 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74589.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78266.log 2026-02-19T21:12:54.445 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74589.log.gz 2026-02-19T21:12:54.445 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125562.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96402.log 2026-02-19T21:12:54.445 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78266.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.125562.log.gz 2026-02-19T21:12:54.445 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108641.log 2026-02-19T21:12:54.445 INFO:teuthology.orchestra.run.trial151.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78266.log.gz 2026-02-19T21:12:54.446 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101475.log 2026-02-19T21:12:54.446 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96402.log: /var/log/ceph/cluster2-client.mirror.108641.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30329.log 2026-02-19T21:12:54.446 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.108641.log.gz 2026-02-19T21:12:54.446 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106479.log 2026-02-19T21:12:54.446 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101475.log.gz 2026-02-19T21:12:54.447 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30329.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109890.log 2026-02-19T21:12:54.447 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75556.log 2026-02-19T21:12:54.447 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30329.log.gz 2026-02-19T21:12:54.447 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106479.log.gz 2026-02-19T21:12:54.447 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.109890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89471.log 2026-02-19T21:12:54.448 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75556.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70926.log 2026-02-19T21:12:54.448 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89471.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53387.log 2026-02-19T21:12:54.448 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70926.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.109890.log.gz 2026-02-19T21:12:54.448 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27803.log 2026-02-19T21:12:54.448 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.70926.log.gz 2026-02-19T21:12:54.449 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53387.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117999.log 2026-02-19T21:12:54.449 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53387.log.gz 2026-02-19T21:12:54.449 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27803.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22995.log 2026-02-19T21:12:54.449 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117999.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27803.log.gz 2026-02-19T21:12:54.449 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85995.log 2026-02-19T21:12:54.449 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.117999.log.gz 2026-02-19T21:12:54.450 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75715.log 2026-02-19T21:12:54.450 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24346.log 2026-02-19T21:12:54.450 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22995.log.gz 2026-02-19T21:12:54.450 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75715.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59090.log 2026-02-19T21:12:54.451 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75556.log.gz 2026-02-19T21:12:54.451 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78163.log 2026-02-19T21:12:54.451 INFO:teuthology.orchestra.run.trial151.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85995.log.gz 2026-02-19T21:12:54.451 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.110130.log 2026-02-19T21:12:54.451 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24346.log.gz 2026-02-19T21:12:54.451 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59090.log: /var/log/ceph/cluster2-client.mirror.78163.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113131.log 2026-02-19T21:12:54.452 INFO:teuthology.orchestra.run.trial151.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78163.log.gz 2026-02-19T21:12:54.452 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59090.log.gz 2026-02-19T21:12:54.452 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.1.110130.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96335.log 2026-02-19T21:12:54.452 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.113131.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96646.log 2026-02-19T21:12:54.453 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.96335.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62524.log 2026-02-19T21:12:54.453 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.113131.log.gz 2026-02-19T21:12:54.453 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96646.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35094.log 2026-02-19T21:12:54.453 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.96335.log.gz 2026-02-19T21:12:54.453 INFO:teuthology.orchestra.run.trial151.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.62524.log: -- replaced with /var/log/ceph/cluster2-client.mirror.96646.log.gz 2026-02-19T21:12:54.453 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65049.log 2026-02-19T21:12:54.453 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62524.log.gz 2026-02-19T21:12:54.453 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.35094.log: -- /var/log/ceph/cluster2-client.mirror.86764.log 2026-02-19T21:12:54.453 INFO:teuthology.orchestra.run.trial151.stderr: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75715.log.gz 2026-02-19T21:12:54.454 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105514.log 2026-02-19T21:12:54.454 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35094.log.gz 2026-02-19T21:12:54.454 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86764.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102488.log 2026-02-19T21:12:54.454 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86764.log.gz 2026-02-19T21:12:54.454 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.105514.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121351.log 2026-02-19T21:12:54.455 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.102488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86468.log 2026-02-19T21:12:54.455 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121351.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.102488.log.gz 2026-02-19T21:12:54.455 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123826.log 2026-02-19T21:12:54.455 INFO:teuthology.orchestra.run.trial151.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.105514.log.gz 2026-02-19T21:12:54.455 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86468.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42168.log 2026-02-19T21:12:54.455 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86468.log.gz 2026-02-19T21:12:54.455 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123826.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56894.log 2026-02-19T21:12:54.456 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123826.log.gz 2026-02-19T21:12:54.456 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42168.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82568.log 2026-02-19T21:12:54.456 INFO:teuthology.orchestra.run.trial151.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121351.log.gz 2026-02-19T21:12:54.456 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75414.log 2026-02-19T21:12:54.456 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42168.log.gz 2026-02-19T21:12:54.456 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82568.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56894.log.gz 2026-02-19T21:12:54.456 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58411.log 2026-02-19T21:12:54.457 INFO:teuthology.orchestra.run.trial151.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65049.log.gz 2026-02-19T21:12:54.457 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75414.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126206.log 2026-02-19T21:12:54.457 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58411.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75414.log.gz 2026-02-19T21:12:54.457 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123893.log 2026-02-19T21:12:54.457 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58411.log.gz 2026-02-19T21:12:54.457 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126206.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95006.log 2026-02-19T21:12:54.457 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126206.log.gz 2026-02-19T21:12:54.458 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123893.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99028.log 2026-02-19T21:12:54.458 INFO:teuthology.orchestra.run.trial013.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-19T21:12:54.458 INFO:teuthology.orchestra.run.trial151.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123893.log.gz 2026-02-19T21:12:54.458 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61942.log 2026-02-19T21:12:54.458 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118290.log 2026-02-19T21:12:54.459 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99028.log.gz 2026-02-19T21:12:54.459 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61942.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119734.log 2026-02-19T21:12:54.459 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61942.log.gz 2026-02-19T21:12:54.459 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118290.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.125809.log 2026-02-19T21:12:54.459 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118290.log.gz 2026-02-19T21:12:54.459 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122858.log 2026-02-19T21:12:54.459 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119734.log.gz 2026-02-19T21:12:54.462 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.1.125809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97605.log 2026-02-19T21:12:54.462 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122858.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50285.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122858.log.gz 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97605.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68132.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97605.log.gz 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50285.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114642.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50285.log.gz 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68132.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126972.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68132.log.gz 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114642.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95184.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93359.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126972.log: /var/log/ceph/cluster2-client.mirror.95184.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80699.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.95184.log.gz 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.126972.log.gz 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.114642.log.gz 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93359.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93160.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.1.125809.log.gz 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32287.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80699.log: /var/log/ceph/cluster1-client.mirror.93160.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119216.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32287.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93359.log.gz 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114771.log 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.93160.log.gz 2026-02-19T21:12:54.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119216.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32287.log.gz 2026-02-19T21:12:54.464 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122144.log 2026-02-19T21:12:54.464 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114771.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89175.log 2026-02-19T21:12:54.464 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114771.log.gz 2026-02-19T21:12:54.464 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101050.log 2026-02-19T21:12:54.465 INFO:teuthology.orchestra.run.trial151.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119216.log.gz 2026-02-19T21:12:54.465 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126073.log 2026-02-19T21:12:54.465 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.89175.log.gz 2026-02-19T21:12:54.465 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42787.log 2026-02-19T21:12:54.465 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126073.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.101050.log.gz 2026-02-19T21:12:54.465 INFO:teuthology.orchestra.run.trial151.stderr: 90.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111908.log 2026-02-19T21:12:54.465 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.122144.log.gz 2026-02-19T21:12:54.466 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42787.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77876.log 2026-02-19T21:12:54.466 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111908.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42787.log.gz 2026-02-19T21:12:54.466 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116610.log 2026-02-19T21:12:54.467 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.111908.log.gz/var/log/ceph/cluster2-client.mirror.77876.log: 2026-02-19T21:12:54.467 INFO:teuthology.orchestra.run.trial151.stderr: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77876.log.gz 2026-02-19T21:12:54.467 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44692.log 2026-02-19T21:12:54.467 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45375.log 2026-02-19T21:12:54.467 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116610.log: /var/log/ceph/cluster2-client.mirror.44692.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23386.log 2026-02-19T21:12:54.468 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45375.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44692.log.gz 2026-02-19T21:12:54.468 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45375.log.gz 2026-02-19T21:12:54.468 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50320.log 2026-02-19T21:12:54.468 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23386.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112796.log 2026-02-19T21:12:54.469 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50320.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116610.log.gz 2026-02-19T21:12:54.469 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23386.log.gz 2026-02-19T21:12:54.469 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.50320.log.gz 2026-02-19T21:12:54.469 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54760.log 2026-02-19T21:12:54.470 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112796.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86132.log 2026-02-19T21:12:54.470 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112796.log.gz 2026-02-19T21:12:54.470 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54760.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77183.log 2026-02-19T21:12:54.470 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54760.log.gz 2026-02-19T21:12:54.470 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44881.log 2026-02-19T21:12:54.471 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86132.log: /var/log/ceph/cluster1-client.mirror.77183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124984.log 2026-02-19T21:12:54.471 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.86132.log.gz 2026-02-19T21:12:54.471 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44881.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116652.log 2026-02-19T21:12:54.472 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.124984.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.124984.log.gz 2026-02-19T21:12:54.472 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92370.log 2026-02-19T21:12:54.474 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116652.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.77183.log.gz 2026-02-19T21:12:54.474 INFO:teuthology.orchestra.run.trial151.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44881.log.gz 2026-02-19T21:12:54.475 INFO:teuthology.orchestra.run.trial151.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.110130.log.gz 2026-02-19T21:12:54.475 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33480.log 2026-02-19T21:12:54.475 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128736.log 2026-02-19T21:12:54.476 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92370.log.gz 2026-02-19T21:12:54.476 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128461.log 2026-02-19T21:12:54.476 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93522.log 2026-02-19T21:12:54.476 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128736.log: /var/log/ceph/cluster2-client.admin.128461.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128736.log.gz 2026-02-19T21:12:54.476 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33480.log.gz 2026-02-19T21:12:54.476 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128461.log.gz 2026-02-19T21:12:54.477 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127154.log 2026-02-19T21:12:54.477 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89337.log 2026-02-19T21:12:54.477 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93522.log: /var/log/ceph/cluster1-client.mirror.127154.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86364.log 2026-02-19T21:12:54.477 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127154.log.gz 2026-02-19T21:12:54.477 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93522.log.gz 2026-02-19T21:12:54.478 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89337.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18613.log 2026-02-19T21:12:54.478 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89337.log.gz 2026-02-19T21:12:54.478 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111607.log 2026-02-19T21:12:54.478 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86364.log: /var/log/ceph/cluster2-client.admin.18613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83360.log 2026-02-19T21:12:54.478 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18613.log.gz 2026-02-19T21:12:54.479 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111607.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111607.log.gz 2026-02-19T21:12:54.480 INFO:teuthology.orchestra.run.trial151.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86364.log.gz 2026-02-19T21:12:54.481 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93334.log 2026-02-19T21:12:54.481 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53743.log 2026-02-19T21:12:54.481 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.83360.log: /var/log/ceph/cluster1-client.mirror.93334.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122636.log 2026-02-19T21:12:54.481 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.93334.log.gz 2026-02-19T21:12:54.482 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53743.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53743.log.gz 2026-02-19T21:12:54.482 INFO:teuthology.orchestra.run.trial151.stderr: 94.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83360.log.gz 2026-02-19T21:12:54.482 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53290.log 2026-02-19T21:12:54.483 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120526.log 2026-02-19T21:12:54.483 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122636.log: /var/log/ceph/cluster2-client.mirror.53290.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.122636.log.gz 2026-02-19T21:12:54.483 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124141.log 2026-02-19T21:12:54.483 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53290.log.gz 2026-02-19T21:12:54.483 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120526.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47468.log 2026-02-19T21:12:54.483 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74492.log 2026-02-19T21:12:54.484 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124141.log: 89.4%/var/log/ceph/cluster2-client.mirror.47468.log: -- replaced with /var/log/ceph/cluster2-client.mirror.120526.log.gz 2026-02-19T21:12:54.484 INFO:teuthology.orchestra.run.trial151.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.124141.log.gz 2026-02-19T21:12:54.484 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47468.log.gz 2026-02-19T21:12:54.484 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121079.log 2026-02-19T21:12:54.484 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30267.log 2026-02-19T21:12:54.485 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74492.log: /var/log/ceph/cluster2-client.mirror.121079.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24893.log 2026-02-19T21:12:54.485 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74492.log.gz 2026-02-19T21:12:54.485 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30267.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61713.log 2026-02-19T21:12:54.486 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24893.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30267.log.gz 2026-02-19T21:12:54.486 INFO:teuthology.orchestra.run.trial151.stderr: 90.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22902.log 2026-02-19T21:12:54.486 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.121079.log.gz 2026-02-19T21:12:54.486 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.24893.log.gz 2026-02-19T21:12:54.486 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61713.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30170.log 2026-02-19T21:12:54.487 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61713.log.gz 2026-02-19T21:12:54.487 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107405.log 2026-02-19T21:12:54.487 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22902.log: /var/log/ceph/cluster2-client.mirror.30170.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98541.log 2026-02-19T21:12:54.487 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107405.log: 88.3% 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30170.log.gz 2026-02-19T21:12:54.487 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.22902.log.gz 2026-02-19T21:12:54.488 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107405.log.gz 2026-02-19T21:12:54.488 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34900.log 2026-02-19T21:12:54.488 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98541.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38319.log 2026-02-19T21:12:54.488 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98541.log.gz 2026-02-19T21:12:54.488 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117582.log 2026-02-19T21:12:54.489 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34900.log: /var/log/ceph/cluster2-client.mirror.38319.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63882.log 2026-02-19T21:12:54.489 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117582.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.34900.log.gz 2026-02-19T21:12:54.489 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38319.log.gz 2026-02-19T21:12:54.489 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104313.log 2026-02-19T21:12:54.490 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63882.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63882.log.gz 2026-02-19T21:12:54.490 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80676.log 2026-02-19T21:12:54.491 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104313.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103807.log 2026-02-19T21:12:54.491 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104313.log.gz 2026-02-19T21:12:54.491 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80676.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80676.log.gz 2026-02-19T21:12:54.491 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120619.log 2026-02-19T21:12:54.492 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103807.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25122.log 2026-02-19T21:12:54.492 INFO:teuthology.orchestra.run.trial151.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117582.log.gz 2026-02-19T21:12:54.492 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120619.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.120619.log.gz 2026-02-19T21:12:54.493 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31608.log 2026-02-19T21:12:54.493 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25122.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114662.log 2026-02-19T21:12:54.494 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31608.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25122.log.gz 2026-02-19T21:12:54.494 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36002.log 2026-02-19T21:12:54.494 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.114662.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31608.log.gz 2026-02-19T21:12:54.494 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124118.log 2026-02-19T21:12:54.495 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36002.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.114662.log.gz 2026-02-19T21:12:54.495 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36002.log.gz 2026-02-19T21:12:54.495 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124748.log 2026-02-19T21:12:54.496 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88171.log 2026-02-19T21:12:54.496 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124118.log: /var/log/ceph/cluster1-client.mirror.124748.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124118.log.gz 2026-02-19T21:12:54.496 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124748.log.gz 2026-02-19T21:12:54.496 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115501.log 2026-02-19T21:12:54.496 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65356.log 2026-02-19T21:12:54.497 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88171.log: /var/log/ceph/cluster2-client.mirror.115501.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88171.log.gz 2026-02-19T21:12:54.497 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115501.log.gz 2026-02-19T21:12:54.497 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81740.log 2026-02-19T21:12:54.497 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112059.log 2026-02-19T21:12:54.498 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.65356.log: /var/log/ceph/cluster2-client.mirror.81740.log: 89.2% -- replaced with /var/log/ceph/cluster1-client.admin.65356.log.gz 2026-02-19T21:12:54.498 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.93160.log 2026-02-19T21:12:54.499 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112059.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112059.log.gz 2026-02-19T21:12:54.499 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77927.log 2026-02-19T21:12:54.506 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.1.93160.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.1.93160.log.gz 2026-02-19T21:12:54.506 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40942.log 2026-02-19T21:12:54.506 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77927.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77927.log.gz 2026-02-19T21:12:54.507 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107861.log 2026-02-19T21:12:54.507 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40942.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40942.log.gz 2026-02-19T21:12:54.507 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116585.log 2026-02-19T21:12:54.508 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107861.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107861.log.gz 2026-02-19T21:12:54.508 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121309.log 2026-02-19T21:12:54.509 INFO:teuthology.orchestra.run.trial013.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-19T21:12:54.509 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.116585.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.116585.log.gz 2026-02-19T21:12:54.509 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89823.log 2026-02-19T21:12:54.510 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121309.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.99772.log.gz 2026-02-19T21:12:54.510 INFO:teuthology.orchestra.run.trial151.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.121309.log.gz 2026-02-19T21:12:54.510 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97493.log 2026-02-19T21:12:54.510 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89823.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72482.log 2026-02-19T21:12:54.511 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.89823.log.gz 2026-02-19T21:12:54.511 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97493.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97493.log.gz 2026-02-19T21:12:54.511 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98369.log 2026-02-19T21:12:54.512 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121753.log 2026-02-19T21:12:54.512 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72482.log: /var/log/ceph/cluster2-client.mirror.98369.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72482.log.gz 2026-02-19T21:12:54.512 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69280.log 2026-02-19T21:12:54.513 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121753.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121753.log.gz 2026-02-19T21:12:54.513 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52316.log 2026-02-19T21:12:54.514 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69280.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69280.log.gz 2026-02-19T21:12:54.514 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56277.log 2026-02-19T21:12:54.515 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52316.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52316.log.gz 2026-02-19T21:12:54.515 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107251.log 2026-02-19T21:12:54.516 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56277.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56277.log.gz 2026-02-19T21:12:54.516 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125522.log 2026-02-19T21:12:54.516 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107251.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107251.log.gz 2026-02-19T21:12:54.517 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18873.log 2026-02-19T21:12:54.518 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125522.log: 95.9% 97.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98369.log.gz 2026-02-19T21:12:54.518 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.103807.log.gz 2026-02-19T21:12:54.518 INFO:teuthology.orchestra.run.trial151.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.125522.log.gz 2026-02-19T21:12:54.519 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91215.log 2026-02-19T21:12:54.519 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35614.log 2026-02-19T21:12:54.519 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18873.log: /var/log/ceph/cluster2-client.mirror.91215.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18873.log.gz 2026-02-19T21:12:54.520 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66386.log 2026-02-19T21:12:54.520 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.91215.log.gz 2026-02-19T21:12:54.520 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35676.log 2026-02-19T21:12:54.520 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38866.log 2026-02-19T21:12:54.520 INFO:teuthology.orchestra.run.trial151.stderr: 88.6%/var/log/ceph/cluster2-client.mirror.66386.log: -- replaced with /var/log/ceph/cluster2-client.mirror.35614.log.gz 2026-02-19T21:12:54.521 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35676.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66386.log.gz 2026-02-19T21:12:54.521 INFO:teuthology.orchestra.run.trial151.stderr: 88.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88603.log 2026-02-19T21:12:54.521 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.35676.log.gz 2026-02-19T21:12:54.521 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38866.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95246.log 2026-02-19T21:12:54.521 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88603.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116898.log 2026-02-19T21:12:54.522 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95246.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88603.log.gz 2026-02-19T21:12:54.522 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38866.log.gz 2026-02-19T21:12:54.522 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95246.log.gz 2026-02-19T21:12:54.522 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68291.log 2026-02-19T21:12:54.522 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98858.log 2026-02-19T21:12:54.523 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68291.log: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26674.log 2026-02-19T21:12:54.523 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.116898.log.gz 2026-02-19T21:12:54.523 INFO:teuthology.orchestra.run.trial151.stderr: 89.0%/var/log/ceph/cluster2-client.mirror.98858.log: -- replaced with /var/log/ceph/cluster2-client.mirror.68291.log.gz 2026-02-19T21:12:54.523 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98767.log 2026-02-19T21:12:54.524 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117265.log 2026-02-19T21:12:54.524 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26674.log: /var/log/ceph/cluster1-client.mirror.98767.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.98767.log.gz 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26674.log.gz 2026-02-19T21:12:54.524 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:54.524 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126356.log 2026-02-19T21:12:54.525 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87251.log 2026-02-19T21:12:54.525 INFO:teuthology.orchestra.run.trial151.stderr: 92.8%/var/log/ceph/cluster1-client.mirror.117265.log: -- replaced with /var/log/ceph/cluster2-client.mirror.98858.log.gz 2026-02-19T21:12:54.526 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126356.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.117265.log.gz 2026-02-19T21:12:54.526 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32384.log 2026-02-19T21:12:54.527 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29747.log 2026-02-19T21:12:54.527 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87251.log: /var/log/ceph/cluster2-client.mirror.32384.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.87251.log.gz 2026-02-19T21:12:54.527 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25898.log 2026-02-19T21:12:54.527 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32384.log.gz 2026-02-19T21:12:54.528 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29747.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.29747.log.gz 2026-02-19T21:12:54.528 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22437.log 2026-02-19T21:12:54.528 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111674.log 2026-02-19T21:12:54.529 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25898.log: /var/log/ceph/cluster2-client.mirror.22437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99133.log 2026-02-19T21:12:54.529 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.22437.log.gz 2026-02-19T21:12:54.529 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25898.log.gz 2026-02-19T21:12:54.530 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.111674.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.111674.log.gz 2026-02-19T21:12:54.537 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42362.log 2026-02-19T21:12:54.537 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66448.log 2026-02-19T21:12:54.538 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99133.log: /var/log/ceph/cluster2-client.mirror.42362.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99133.log.gz 2026-02-19T21:12:54.538 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83470.log 2026-02-19T21:12:54.538 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42362.log.gz 2026-02-19T21:12:54.538 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66448.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109146.log 2026-02-19T21:12:54.539 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83470.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66448.log.gz 2026-02-19T21:12:54.539 INFO:teuthology.orchestra.run.trial151.stderr: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.83470.log.gz 2026-02-19T21:12:54.539 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108548.log 2026-02-19T21:12:54.539 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127803.log 2026-02-19T21:12:54.540 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109146.log: /var/log/ceph/cluster2-client.mirror.108548.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109146.log.gz 2026-02-19T21:12:54.540 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108548.log.gz 2026-02-19T21:12:54.540 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47406.log 2026-02-19T21:12:54.540 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60646.log 2026-02-19T21:12:54.540 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.127803.log: /var/log/ceph/cluster2-client.mirror.47406.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127803.log.gz 2026-02-19T21:12:54.541 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.47406.log.gz 2026-02-19T21:12:54.541 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76925.log 2026-02-19T21:12:54.541 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60646.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60646.log.gz 2026-02-19T21:12:54.542 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110509.log 2026-02-19T21:12:54.542 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21635.log 2026-02-19T21:12:54.542 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76925.log: /var/log/ceph/cluster2-client.mirror.110509.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.110509.log.gz 2026-02-19T21:12:54.545 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.76925.log.gz 2026-02-19T21:12:54.546 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119032.log 2026-02-19T21:12:54.546 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21635.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82639.log 2026-02-19T21:12:54.546 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119032.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21635.log.gz 2026-02-19T21:12:54.546 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61131.log 2026-02-19T21:12:54.547 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82639.log: 89.2% 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119032.log.gz 2026-02-19T21:12:54.547 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.82639.log.gz 2026-02-19T21:12:54.547 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96696.log 2026-02-19T21:12:54.547 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113335.log 2026-02-19T21:12:54.548 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61131.log: /var/log/ceph/cluster2-client.mirror.96696.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61131.log.gz 2026-02-19T21:12:54.548 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96696.log.gz 2026-02-19T21:12:54.548 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120548.log 2026-02-19T21:12:54.548 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76327.log 2026-02-19T21:12:54.549 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113335.log: /var/log/ceph/cluster2-client.mirror.120548.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113335.log.gz 2026-02-19T21:12:54.549 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.120548.log.gz 2026-02-19T21:12:54.557 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108860.log 2026-02-19T21:12:54.557 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76327.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106207.log 2026-02-19T21:12:54.558 INFO:teuthology.orchestra.run.trial151.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76327.log.gz 2026-02-19T21:12:54.558 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108860.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18795.log 2026-02-19T21:12:54.558 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108860.log.gz 2026-02-19T21:12:54.558 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106207.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52805.log 2026-02-19T21:12:54.559 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18795.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18795.log.gz 2026-02-19T21:12:54.559 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55051.log 2026-02-19T21:12:54.559 INFO:teuthology.orchestra.run.trial151.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106207.log.gz/var/log/ceph/cluster2-client.mirror.52805.log: 2026-02-19T21:12:54.560 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52805.log.gz 2026-02-19T21:12:54.560 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42106.log 2026-02-19T21:12:54.560 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21414.log 2026-02-19T21:12:54.560 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55051.log: /var/log/ceph/cluster2-client.mirror.42106.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55051.log.gz 2026-02-19T21:12:54.561 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120193.log 2026-02-19T21:12:54.561 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42106.log.gz 2026-02-19T21:12:54.561 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53870.log 2026-02-19T21:12:54.562 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21414.log: /var/log/ceph/cluster2-client.mirror.120193.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120193.log.gz 2026-02-19T21:12:54.562 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.21414.log.gz 2026-02-19T21:12:54.562 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49641.log 2026-02-19T21:12:54.562 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53870.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60875.log 2026-02-19T21:12:54.562 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53870.log.gz 2026-02-19T21:12:54.563 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49641.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82200.log 2026-02-19T21:12:54.563 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49641.log.gz 2026-02-19T21:12:54.563 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60875.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76819.log 2026-02-19T21:12:54.563 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60875.log.gz 2026-02-19T21:12:54.566 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.82200.log: 95.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82200.log.gz 2026-02-19T21:12:54.566 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108673.log 2026-02-19T21:12:54.567 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68679.log 2026-02-19T21:12:54.567 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76819.log: /var/log/ceph/cluster1-client.mirror.108673.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.108673.log.gz 2026-02-19T21:12:54.567 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111530.log 2026-02-19T21:12:54.568 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68679.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68679.log.gz 2026-02-19T21:12:54.568 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104853.log 2026-02-19T21:12:54.569 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111530.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.111530.log.gz 2026-02-19T21:12:54.569 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49835.log 2026-02-19T21:12:54.570 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104853.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104853.log.gz 2026-02-19T21:12:54.570 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76819.log.gz 2026-02-19T21:12:54.570 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55210.log 2026-02-19T21:12:54.571 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90469.log 2026-02-19T21:12:54.571 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49835.log: /var/log/ceph/cluster2-client.mirror.55210.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49835.log.gz 2026-02-19T21:12:54.571 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55210.log.gz 2026-02-19T21:12:54.571 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97416.log 2026-02-19T21:12:54.572 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97026.log 2026-02-19T21:12:54.572 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90469.log: /var/log/ceph/cluster2-client.mirror.97416.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90469.log.gz 2026-02-19T21:12:54.572 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97416.log.gz 2026-02-19T21:12:54.572 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93856.log 2026-02-19T21:12:54.574 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.97026.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.97026.log.gz 2026-02-19T21:12:54.574 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25413.log 2026-02-19T21:12:54.574 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90126.log 2026-02-19T21:12:54.575 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93856.log: /var/log/ceph/cluster2-client.mirror.25413.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93856.log.gz 2026-02-19T21:12:54.575 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25413.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24540.log 2026-02-19T21:12:54.575 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:54.575 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90126.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90126.log.gz 2026-02-19T21:12:54.576 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94341.log 2026-02-19T21:12:54.576 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100938.log 2026-02-19T21:12:54.577 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24540.log: /var/log/ceph/cluster2-client.mirror.94341.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94341.log.gz 2026-02-19T21:12:54.577 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24540.log.gz 2026-02-19T21:12:54.577 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95829.log 2026-02-19T21:12:54.577 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70829.log 2026-02-19T21:12:54.577 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100938.log: /var/log/ceph/cluster2-client.mirror.95829.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100938.log.gz 2026-02-19T21:12:54.578 INFO:teuthology.orchestra.run.trial151.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95829.log.gz 2026-02-19T21:12:54.578 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81205.log 2026-02-19T21:12:54.578 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122296.log 2026-02-19T21:12:54.578 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70829.log: /var/log/ceph/cluster2-client.mirror.81205.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70829.log.gz 2026-02-19T21:12:54.579 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122557.log 2026-02-19T21:12:54.580 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122296.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.122296.log.gz 2026-02-19T21:12:54.581 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38199.log 2026-02-19T21:12:54.581 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122557.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.122557.log.gz 2026-02-19T21:12:54.581 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19907.log 2026-02-19T21:12:54.582 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38199.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38199.log.gz 2026-02-19T21:12:54.582 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25607.log 2026-02-19T21:12:54.583 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19907.log: 56.0% -- replaced with /var/log/ceph/cluster1-client.mirror.19907.log.gz 2026-02-19T21:12:54.583 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128632.log 2026-02-19T21:12:54.584 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25607.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25607.log.gz 2026-02-19T21:12:54.584 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124861.log 2026-02-19T21:12:54.584 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128632.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128632.log.gz 2026-02-19T21:12:54.585 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76448.log 2026-02-19T21:12:54.585 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124861.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124861.log.gz 2026-02-19T21:12:54.586 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76905.log 2026-02-19T21:12:54.589 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76448.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.76448.log.gz 2026-02-19T21:12:54.590 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71120.log 2026-02-19T21:12:54.593 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76905.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76905.log.gz 2026-02-19T21:12:54.594 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93379.log 2026-02-19T21:12:54.594 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71120.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71120.log.gz 2026-02-19T21:12:54.594 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89404.log 2026-02-19T21:12:54.596 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.93379.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.93379.log.gz 2026-02-19T21:12:54.596 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74748.log 2026-02-19T21:12:54.597 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.89404.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.89404.log.gz 2026-02-19T21:12:54.597 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54337.log 2026-02-19T21:12:54.598 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74748.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74748.log.gz 2026-02-19T21:12:54.598 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66642.log 2026-02-19T21:12:54.599 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54337.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54337.log.gz 2026-02-19T21:12:54.599 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40519.log 2026-02-19T21:12:54.600 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66642.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66642.log.gz 2026-02-19T21:12:54.600 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101775.log 2026-02-19T21:12:54.601 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40519.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40519.log.gz 2026-02-19T21:12:54.601 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125355.log 2026-02-19T21:12:54.601 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101775.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101775.log.gz 2026-02-19T21:12:54.602 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68873.log 2026-02-19T21:12:54.603 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125355.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.125355.log.gz 2026-02-19T21:12:54.603 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115168.log 2026-02-19T21:12:54.603 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68873.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68873.log.gz 2026-02-19T21:12:54.604 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126648.log 2026-02-19T21:12:54.605 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115168.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.115168.log.gz 2026-02-19T21:12:54.605 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88846.log 2026-02-19T21:12:54.607 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126648.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.126648.log.gz 2026-02-19T21:12:54.607 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126764.log 2026-02-19T21:12:54.608 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88846.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88846.log.gz 2026-02-19T21:12:54.608 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124478.log 2026-02-19T21:12:54.609 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126764.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.126764.log.gz 2026-02-19T21:12:54.609 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120068.log 2026-02-19T21:12:54.610 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124478.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124478.log.gz 2026-02-19T21:12:54.610 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.78978.log 2026-02-19T21:12:54.611 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120068.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120068.log.gz 2026-02-19T21:12:54.611 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114311.log 2026-02-19T21:12:54.621 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.3.78978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126308.log 2026-02-19T21:12:54.622 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114311.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114311.log.gz 2026-02-19T21:12:54.633 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87975.log 2026-02-19T21:12:54.634 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126308.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.126308.log.gz 2026-02-19T21:12:54.641 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41780.log 2026-02-19T21:12:54.643 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87975.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.116652.log.gz 2026-02-19T21:12:54.644 INFO:teuthology.orchestra.run.trial151.stderr: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87975.log.gz 2026-02-19T21:12:54.645 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76991.log 2026-02-19T21:12:54.645 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41780.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109534.log 2026-02-19T21:12:54.646 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41780.log.gz 2026-02-19T21:12:54.649 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76991.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76991.log.gz 2026-02-19T21:12:54.650 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98583.log 2026-02-19T21:12:54.650 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109534.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109534.log.gz 2026-02-19T21:12:54.651 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78483.log 2026-02-19T21:12:54.651 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98583.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98583.log.gz 2026-02-19T21:12:54.652 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83770.log 2026-02-19T21:12:54.666 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78483.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118650.log 2026-02-19T21:12:54.666 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83770.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83770.log.gz 2026-02-19T21:12:54.681 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47503.log 2026-02-19T21:12:54.683 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.118650.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.118650.log.gz 2026-02-19T21:12:54.689 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42203.log 2026-02-19T21:12:54.690 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47503.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47503.log.gz 2026-02-19T21:12:54.691 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.96402.log.gz 2026-02-19T21:12:54.691 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64464.log 2026-02-19T21:12:54.692 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42203.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42203.log.gz 2026-02-19T21:12:54.692 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88272.log 2026-02-19T21:12:54.692 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64464.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64464.log.gz 2026-02-19T21:12:54.693 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64367.log 2026-02-19T21:12:54.693 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88272.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88272.log.gz 2026-02-19T21:12:54.694 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71085.log 2026-02-19T21:12:54.694 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64367.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64367.log.gz 2026-02-19T21:12:54.694 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49897.log 2026-02-19T21:12:54.695 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71085.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71085.log.gz 2026-02-19T21:12:54.695 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103232.log 2026-02-19T21:12:54.695 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49897.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49897.log.gz 2026-02-19T21:12:54.697 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54025.log 2026-02-19T21:12:54.697 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103232.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103232.log.gz 2026-02-19T21:12:54.697 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52867.log 2026-02-19T21:12:54.700 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.54025.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54025.log.gz 2026-02-19T21:12:54.701 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.119339.log 2026-02-19T21:12:54.701 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52867.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52867.log.gz 2026-02-19T21:12:54.701 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124658.log 2026-02-19T21:12:54.706 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.3.119339.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.3.119339.log.gz 2026-02-19T21:12:54.706 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93091.log 2026-02-19T21:12:54.707 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124658.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124658.log.gz 2026-02-19T21:12:54.707 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76003.log 2026-02-19T21:12:54.707 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93091.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93091.log.gz 2026-02-19T21:12:54.708 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126994.log 2026-02-19T21:12:54.708 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76003.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76003.log.gz 2026-02-19T21:12:54.709 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118209.log 2026-02-19T21:12:54.709 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126994.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.126994.log.gz 2026-02-19T21:12:54.710 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28482.log 2026-02-19T21:12:54.710 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118209.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118209.log.gz 2026-02-19T21:12:54.710 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118347.log 2026-02-19T21:12:54.711 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28482.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28482.log.gz 2026-02-19T21:12:54.711 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-19T21:12:54.712 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118347.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118347.log.gz 2026-02-19T21:12:54.712 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85306.log 2026-02-19T21:12:54.713 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97374.log 2026-02-19T21:12:54.725 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.85306.log: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.85306.log.gz 2026-02-19T21:12:54.737 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27997.log 2026-02-19T21:12:54.738 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97374.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97374.log.gz 2026-02-19T21:12:54.745 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52382.log 2026-02-19T21:12:54.753 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27997.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92661.log 2026-02-19T21:12:54.754 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52382.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52382.log.gz 2026-02-19T21:12:54.757 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27997.log.gz 2026-02-19T21:12:54.761 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35032.log 2026-02-19T21:12:54.761 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92661.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.92661.log.gz 2026-02-19T21:12:54.769 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98302.log 2026-02-19T21:12:54.777 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35032.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102941.log 2026-02-19T21:12:54.778 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.98302.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.98302.log.gz 2026-02-19T21:12:54.781 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.35032.log.gz 2026-02-19T21:12:54.782 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.113854.log.gz 2026-02-19T21:12:54.782 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100670.log 2026-02-19T21:12:54.783 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102941.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102941.log.gz 2026-02-19T21:12:54.783 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95954.log 2026-02-19T21:12:54.784 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100670.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100670.log.gz 2026-02-19T21:12:54.784 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75171.log 2026-02-19T21:12:54.786 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.95954.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.95954.log.gz 2026-02-19T21:12:54.786 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77226.log 2026-02-19T21:12:54.787 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75171.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75171.log.gz 2026-02-19T21:12:54.787 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34644.log 2026-02-19T21:12:54.791 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.77226.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.77226.log.gz 2026-02-19T21:12:54.791 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127772.log 2026-02-19T21:12:54.792 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34644.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34644.log.gz 2026-02-19T21:12:54.793 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80974.log 2026-02-19T21:12:54.793 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127772.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127772.log.gz 2026-02-19T21:12:54.793 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94171.log 2026-02-19T21:12:54.808 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80974.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83493.log.gz 2026-02-19T21:12:54.809 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122948.log 2026-02-19T21:12:54.809 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.94171.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.94171.log.gz 2026-02-19T21:12:54.811 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42884.log 2026-02-19T21:12:54.811 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122948.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.122948.log.gz 2026-02-19T21:12:54.811 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124613.log 2026-02-19T21:12:54.811 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42884.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.42884.log.gz 2026-02-19T21:12:54.812 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109278.log 2026-02-19T21:12:54.812 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124613.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124613.log.gz 2026-02-19T21:12:54.813 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77097.log 2026-02-19T21:12:54.813 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109278.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109278.log.gz 2026-02-19T21:12:54.813 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47212.log 2026-02-19T21:12:54.817 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.77097.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.77097.log.gz 2026-02-19T21:12:54.817 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62039.log 2026-02-19T21:12:54.817 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47212.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47212.log.gz 2026-02-19T21:12:54.818 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90203.log 2026-02-19T21:12:54.818 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62039.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62039.log.gz 2026-02-19T21:12:54.819 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38804.log 2026-02-19T21:12:54.819 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.90203.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.90203.log.gz 2026-02-19T21:12:54.820 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52514.log 2026-02-19T21:12:54.821 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38804.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38804.log.gz 2026-02-19T21:12:54.821 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110598.log 2026-02-19T21:12:54.821 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52514.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52514.log.gz 2026-02-19T21:12:54.822 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75531.log 2026-02-19T21:12:54.822 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110598.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110598.log.gz 2026-02-19T21:12:54.823 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45090.log 2026-02-19T21:12:54.823 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.75531.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.75531.log.gz 2026-02-19T21:12:54.824 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68229.log 2026-02-19T21:12:54.837 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45090.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77806.log 2026-02-19T21:12:54.838 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68229.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68229.log.gz 2026-02-19T21:12:54.849 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41330.log 2026-02-19T21:12:54.850 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77806.log: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77806.log.gz 2026-02-19T21:12:54.865 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76643.log 2026-02-19T21:12:54.866 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41330.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41330.log.gz 2026-02-19T21:12:54.868 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82568.log.gz 2026-02-19T21:12:54.869 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67647.log 2026-02-19T21:12:54.872 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76643.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76643.log.gz 2026-02-19T21:12:54.873 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80846.log 2026-02-19T21:12:54.873 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67647.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67647.log.gz 2026-02-19T21:12:54.874 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117448.log 2026-02-19T21:12:54.874 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80846.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80846.log.gz 2026-02-19T21:12:54.875 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120668.log 2026-02-19T21:12:54.875 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117448.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117448.log.gz 2026-02-19T21:12:54.876 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58799.log 2026-02-19T21:12:54.876 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120668.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.120668.log.gz 2026-02-19T21:12:54.876 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123961.log 2026-02-19T21:12:54.877 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58799.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58799.log.gz 2026-02-19T21:12:54.877 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30073.log 2026-02-19T21:12:54.878 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123961.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123961.log.gz 2026-02-19T21:12:54.878 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48050.log 2026-02-19T21:12:54.880 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30073.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30073.log.gz 2026-02-19T21:12:54.880 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39351.log 2026-02-19T21:12:54.880 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48050.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48050.log.gz 2026-02-19T21:12:54.880 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105733.log 2026-02-19T21:12:54.882 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39351.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39351.log.gz 2026-02-19T21:12:54.882 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121108.log 2026-02-19T21:12:54.882 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105733.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105733.log.gz 2026-02-19T21:12:54.882 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78423.log 2026-02-19T21:12:54.883 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121108.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121108.log.gz 2026-02-19T21:12:54.883 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28032.log 2026-02-19T21:12:54.889 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78423.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46686.log 2026-02-19T21:12:54.906 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28032.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28032.log.gz 2026-02-19T21:12:54.910 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21600.log 2026-02-19T21:12:54.910 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46686.log: 85.0% -- replaced with /var/log/ceph/cluster2-client.mirror.46686.log.gz 2026-02-19T21:12:54.912 INFO:teuthology.orchestra.run.trial013.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-19T21:12:54.926 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.95006.log.gz 2026-02-19T21:12:54.927 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119478.log 2026-02-19T21:12:54.928 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21600.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21600.log.gz 2026-02-19T21:12:54.928 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90756.log 2026-02-19T21:12:54.929 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119478.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119478.log.gz 2026-02-19T21:12:54.929 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23873.log 2026-02-19T21:12:54.930 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.90756.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.90756.log.gz 2026-02-19T21:12:54.931 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32093.log 2026-02-19T21:12:54.931 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23873.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87486.log 2026-02-19T21:12:54.938 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115210.log 2026-02-19T21:12:54.938 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87486.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87486.log.gz 2026-02-19T21:12:54.946 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48632.log 2026-02-19T21:12:54.946 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32093.log: /var/log/ceph/cluster2-client.mirror.115210.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32093.log.gz 2026-02-19T21:12:54.957 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19151.log 2026-02-19T21:12:54.958 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48632.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48632.log.gz 2026-02-19T21:12:54.973 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77291.log 2026-02-19T21:12:54.974 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19151.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19151.log.gz 2026-02-19T21:12:54.981 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19125.log 2026-02-19T21:12:54.982 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77291.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77291.log.gz 2026-02-19T21:12:54.997 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58605.log 2026-02-19T21:12:54.998 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19125.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19125.log.gz 2026-02-19T21:12:54.998 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.125610.log.gz 2026-02-19T21:12:55.007 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89471.log.gz 2026-02-19T21:12:55.014 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119267.log 2026-02-19T21:12:55.014 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58605.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58605.log.gz 2026-02-19T21:12:55.014 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67938.log 2026-02-19T21:12:55.014 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119267.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.119267.log.gz 2026-02-19T21:12:55.014 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73716.log 2026-02-19T21:12:55.014 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67938.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67938.log.gz 2026-02-19T21:12:55.015 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32322.log 2026-02-19T21:12:55.015 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73716.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73716.log.gz 2026-02-19T21:12:55.015 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67550.log 2026-02-19T21:12:55.015 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32322.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106249.log 2026-02-19T21:12:55.015 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32322.log.gz 2026-02-19T21:12:55.015 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67550.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67550.log.gz 2026-02-19T21:12:55.015 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89291.log 2026-02-19T21:12:55.026 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106249.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84796.log 2026-02-19T21:12:55.026 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89291.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89291.log.gz 2026-02-19T21:12:55.041 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87127.log 2026-02-19T21:12:55.043 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.84796.log: 95.3% -- replaced with /var/log/ceph/cluster1-client.mirror.84796.log.gz 2026-02-19T21:12:55.053 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117188.log 2026-02-19T21:12:55.054 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87127.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.87127.log.gz 2026-02-19T21:12:55.063 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.108469.log.gz 2026-02-19T21:12:55.065 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60289.log 2026-02-19T21:12:55.066 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117188.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.117188.log.gz 2026-02-19T21:12:55.067 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33189.log 2026-02-19T21:12:55.067 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60289.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60289.log.gz 2026-02-19T21:12:55.067 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102238.log 2026-02-19T21:12:55.068 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33189.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33189.log.gz 2026-02-19T21:12:55.068 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53982.log 2026-02-19T21:12:55.069 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102238.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102238.log.gz 2026-02-19T21:12:55.069 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56700.log 2026-02-19T21:12:55.073 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.53982.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53982.log.gz 2026-02-19T21:12:55.073 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45825.log 2026-02-19T21:12:55.074 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56700.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56700.log.gz 2026-02-19T21:12:55.074 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105066.log 2026-02-19T21:12:55.075 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45825.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45825.log.gz 2026-02-19T21:12:55.075 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81614.log 2026-02-19T21:12:55.076 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105066.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105066.log.gz 2026-02-19T21:12:55.076 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80942.log 2026-02-19T21:12:55.086 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88136.log 2026-02-19T21:12:55.087 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80942.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80942.log.gz 2026-02-19T21:12:55.093 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70150.log 2026-02-19T21:12:55.094 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88136.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88136.log.gz 2026-02-19T21:12:55.099 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.126356.log.gz 2026-02-19T21:12:55.101 INFO:teuthology.orchestra.run.trial151.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81614.log.gz 2026-02-19T21:12:55.101 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74395.log 2026-02-19T21:12:55.102 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70150.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101015.log 2026-02-19T21:12:55.102 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.70150.log.gz 2026-02-19T21:12:55.103 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74395.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74395.log.gz 2026-02-19T21:12:55.103 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115424.log 2026-02-19T21:12:55.104 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63168.log 2026-02-19T21:12:55.104 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101015.log.gz 2026-02-19T21:12:55.104 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115424.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37526.log 2026-02-19T21:12:55.106 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63168.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115424.log.gz 2026-02-19T21:12:55.106 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63168.log.gz 2026-02-19T21:12:55.106 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36937.log 2026-02-19T21:12:55.106 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23030.log 2026-02-19T21:12:55.106 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.37526.log: /var/log/ceph/cluster2-client.mirror.36937.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.37526.log.gz 2026-02-19T21:12:55.106 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36937.log.gz 2026-02-19T21:12:55.106 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68617.log 2026-02-19T21:12:55.107 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23030.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.23030.log.gz 2026-02-19T21:12:55.107 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19587.log 2026-02-19T21:12:55.108 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68617.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68617.log.gz 2026-02-19T21:12:55.108 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122205.log 2026-02-19T21:12:55.109 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19587.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43466.log 2026-02-19T21:12:55.109 INFO:teuthology.orchestra.run.trial151.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.mirror.19587.log.gz 2026-02-19T21:12:55.109 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122205.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.122205.log.gz 2026-02-19T21:12:55.110 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66677.log 2026-02-19T21:12:55.110 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43466.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43466.log.gz 2026-02-19T21:12:55.110 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52417.log 2026-02-19T21:12:55.111 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66677.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66677.log.gz 2026-02-19T21:12:55.111 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72253.log 2026-02-19T21:12:55.111 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52417.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52417.log.gz 2026-02-19T21:12:55.112 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36646.log 2026-02-19T21:12:55.112 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72253.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72253.log.gz 2026-02-19T21:12:55.112 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64235.log 2026-02-19T21:12:55.113 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36646.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.115210.log.gz 2026-02-19T21:12:55.115 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36646.log.gz 2026-02-19T21:12:55.115 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22472.log 2026-02-19T21:12:55.115 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64235.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77745.log 2026-02-19T21:12:55.115 INFO:teuthology.orchestra.run.trial151.stderr: 89.0%/var/log/ceph/cluster2-client.mirror.22472.log: -- replaced with /var/log/ceph/cluster2-client.mirror.64235.log.gz 2026-02-19T21:12:55.115 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76600.log 2026-02-19T21:12:55.115 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.77745.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.22472.log.gz 2026-02-19T21:12:55.115 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67162.log 2026-02-19T21:12:55.115 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77745.log.gz/var/log/ceph/cluster2-client.mirror.76600.log: 2026-02-19T21:12:55.116 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122813.log 2026-02-19T21:12:55.117 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67162.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67162.log.gz 2026-02-19T21:12:55.117 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110814.log 2026-02-19T21:12:55.117 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122813.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.122813.log.gz 2026-02-19T21:12:55.117 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54372.log 2026-02-19T21:12:55.117 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110814.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110814.log.gz 2026-02-19T21:12:55.118 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86097.log 2026-02-19T21:12:55.118 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54372.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54372.log.gz 2026-02-19T21:12:55.118 INFO:teuthology.orchestra.run.trial151.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76600.log.gz 2026-02-19T21:12:55.118 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44110.log 2026-02-19T21:12:55.118 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86097.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27741.log 2026-02-19T21:12:55.118 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44110.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127196.log 2026-02-19T21:12:55.119 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86097.log.gz 2026-02-19T21:12:55.119 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27741.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92416.log 2026-02-19T21:12:55.119 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44110.log.gz 2026-02-19T21:12:55.119 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127196.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27741.log.gz 2026-02-19T21:12:55.119 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.127196.log.gz 2026-02-19T21:12:55.120 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61166.log 2026-02-19T21:12:55.120 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92416.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89210.log 2026-02-19T21:12:55.120 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92416.log.gz 2026-02-19T21:12:55.120 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128222.log 2026-02-19T21:12:55.121 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61166.log: /var/log/ceph/cluster2-client.mirror.89210.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61166.log.gz 2026-02-19T21:12:55.121 INFO:teuthology.orchestra.run.trial151.stderr: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60192.log 2026-02-19T21:12:55.121 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.89210.log.gz 2026-02-19T21:12:55.122 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.128222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111257.log 2026-02-19T21:12:55.122 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128222.log.gz 2026-02-19T21:12:55.122 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43660.log 2026-02-19T21:12:55.124 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60192.log: 89.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108335.log 2026-02-19T21:12:55.124 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.60192.log.gz 2026-02-19T21:12:55.124 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111257.log: /var/log/ceph/cluster2-client.mirror.43660.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111257.log.gz 2026-02-19T21:12:55.124 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20874.log 2026-02-19T21:12:55.124 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43660.log.gz 2026-02-19T21:12:55.124 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108335.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108335.log.gz 2026-02-19T21:12:55.126 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77438.log 2026-02-19T21:12:55.126 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20874.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124591.log 2026-02-19T21:12:55.126 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77438.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51148.log 2026-02-19T21:12:55.126 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124591.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77438.log.gz 2026-02-19T21:12:55.126 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124591.log.gz 2026-02-19T21:12:55.126 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27450.log 2026-02-19T21:12:55.128 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51148.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78802.log.gz 2026-02-19T21:12:55.129 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68388.log 2026-02-19T21:12:55.129 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27450.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122225.log 2026-02-19T21:12:55.129 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27450.log.gz/var/log/ceph/cluster2-client.mirror.68388.log: 2026-02-19T21:12:55.130 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68388.log.gz 2026-02-19T21:12:55.133 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42849.log 2026-02-19T21:12:55.133 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45472.log 2026-02-19T21:12:55.134 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122225.log: /var/log/ceph/cluster2-client.mirror.42849.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.42849.log.gz 2026-02-19T21:12:55.134 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54178.log 2026-02-19T21:12:55.135 INFO:teuthology.orchestra.run.trial151.stderr: 90.7% 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126073.log.gz 2026-02-19T21:12:55.135 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.122225.log.gz 2026-02-19T21:12:55.135 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45472.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45472.log.gz 2026-02-19T21:12:55.136 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111974.log 2026-02-19T21:12:55.136 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59028.log 2026-02-19T21:12:55.136 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54178.log: /var/log/ceph/cluster2-client.mirror.111974.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76968.log 2026-02-19T21:12:55.137 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59028.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111974.log.gz 2026-02-19T21:12:55.137 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59028.log.gz 2026-02-19T21:12:55.137 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107046.log 2026-02-19T21:12:55.140 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76968.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.76968.log.gz 2026-02-19T21:12:55.140 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15994.log 2026-02-19T21:12:55.141 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107046.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37332.log 2026-02-19T21:12:55.141 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107046.log.gz 2026-02-19T21:12:55.141 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.15994.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15994.log.gz 2026-02-19T21:12:55.141 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25087.log 2026-02-19T21:12:55.141 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118748.log 2026-02-19T21:12:55.142 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25087.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37332.log.gz 2026-02-19T21:12:55.142 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88811.log 2026-02-19T21:12:55.142 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118748.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25087.log.gz 2026-02-19T21:12:55.143 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79825.log 2026-02-19T21:12:55.143 INFO:teuthology.orchestra.run.trial151.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118748.log.gz 2026-02-19T21:12:55.143 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88811.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88811.log.gz 2026-02-19T21:12:55.143 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120762.log 2026-02-19T21:12:55.144 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79825.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42544.log 2026-02-19T21:12:55.144 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120762.log: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.120762.log.gz 2026-02-19T21:12:55.145 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56991.log 2026-02-19T21:12:55.149 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.42544.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42544.log.gz 2026-02-19T21:12:55.150 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45569.log 2026-02-19T21:12:55.150 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56991.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56991.log.gz 2026-02-19T21:12:55.150 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56797.log 2026-02-19T21:12:55.151 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45569.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45569.log.gz 2026-02-19T21:12:55.151 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.77986.log 2026-02-19T21:12:55.152 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56797.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56797.log.gz 2026-02-19T21:12:55.152 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44901.log 2026-02-19T21:12:55.165 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.3.77986.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41683.log 2026-02-19T21:12:55.169 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.44901.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44901.log.gz 2026-02-19T21:12:55.171 INFO:teuthology.orchestra.run.trial151.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.3.77986.log.gz 2026-02-19T21:12:55.173 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126870.log 2026-02-19T21:12:55.174 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41683.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41683.log.gz 2026-02-19T21:12:55.174 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102371.log 2026-02-19T21:12:55.175 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126870.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126870.log.gz 2026-02-19T21:12:55.175 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124681.log 2026-02-19T21:12:55.176 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102371.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102371.log.gz 2026-02-19T21:12:55.176 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29879.log 2026-02-19T21:12:55.177 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124681.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124681.log.gz 2026-02-19T21:12:55.177 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110317.log 2026-02-19T21:12:55.178 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29879.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29879.log.gz 2026-02-19T21:12:55.178 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79426.log 2026-02-19T21:12:55.179 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110317.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110317.log.gz 2026-02-19T21:12:55.179 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27415.log 2026-02-19T21:12:55.182 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79426.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77899.log.gz 2026-02-19T21:12:55.194 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54531.log 2026-02-19T21:12:55.195 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27415.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113524.log 2026-02-19T21:12:55.195 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27415.log.gz 2026-02-19T21:12:55.196 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54531.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54531.log.gz 2026-02-19T21:12:55.196 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25995.log 2026-02-19T21:12:55.196 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113524.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113524.log.gz 2026-02-19T21:12:55.197 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71799.log 2026-02-19T21:12:55.197 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80869.log 2026-02-19T21:12:55.198 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25995.log.gz 2026-02-19T21:12:55.198 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71799.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71799.log.gz 2026-02-19T21:12:55.198 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117786.log 2026-02-19T21:12:55.207 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80869.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82881.log.gz 2026-02-19T21:12:55.208 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108895.log 2026-02-19T21:12:55.208 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117786.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.117786.log.gz 2026-02-19T21:12:55.209 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108718.log 2026-02-19T21:12:55.209 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108895.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.108895.log.gz 2026-02-19T21:12:55.209 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40228.log 2026-02-19T21:12:55.210 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.108718.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.108718.log.gz 2026-02-19T21:12:55.211 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120213.log 2026-02-19T21:12:55.212 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40228.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40228.log.gz 2026-02-19T21:12:55.212 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101810.log 2026-02-19T21:12:55.213 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120213.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.120213.log.gz 2026-02-19T21:12:55.214 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65114.log 2026-02-19T21:12:55.214 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101810.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.101810.log.gz 2026-02-19T21:12:55.214 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31837.log 2026-02-19T21:12:55.228 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65114.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81205.log.gz 2026-02-19T21:12:55.229 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31317.log 2026-02-19T21:12:55.231 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31837.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31837.log.gz 2026-02-19T21:12:55.231 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125087.log 2026-02-19T21:12:55.232 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31317.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.31317.log.gz 2026-02-19T21:12:55.232 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79756.log 2026-02-19T21:12:55.232 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81740.log.gz 2026-02-19T21:12:55.233 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125087.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.125087.log.gz 2026-02-19T21:12:55.234 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99928.log 2026-02-19T21:12:55.234 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79756.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103990.log 2026-02-19T21:12:55.234 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99928.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99928.log.gz 2026-02-19T21:12:55.246 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122657.log 2026-02-19T21:12:55.247 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.103990.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.103990.log.gz 2026-02-19T21:12:55.247 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78879.log 2026-02-19T21:12:55.247 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122657.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.122657.log.gz 2026-02-19T21:12:55.247 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97339.log 2026-02-19T21:12:55.253 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78879.log: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78423.log.gz 2026-02-19T21:12:55.254 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93267.log 2026-02-19T21:12:55.254 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97339.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97339.log.gz 2026-02-19T21:12:55.254 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107081.log 2026-02-19T21:12:55.256 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93267.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.93267.log.gz 2026-02-19T21:12:55.256 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61422.log 2026-02-19T21:12:55.256 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107081.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107081.log.gz 2026-02-19T21:12:55.256 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63300.log 2026-02-19T21:12:55.257 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61422.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61422.log.gz 2026-02-19T21:12:55.257 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50223.log 2026-02-19T21:12:55.258 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63300.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63300.log.gz 2026-02-19T21:12:55.259 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35967.log 2026-02-19T21:12:55.259 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50223.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50223.log.gz 2026-02-19T21:12:55.259 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126563.log 2026-02-19T21:12:55.260 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35967.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35967.log.gz 2026-02-19T21:12:55.260 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39545.log 2026-02-19T21:12:55.260 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126563.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126563.log.gz 2026-02-19T21:12:55.260 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37247.log 2026-02-19T21:12:55.261 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39545.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39545.log.gz 2026-02-19T21:12:55.261 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55695.log 2026-02-19T21:12:55.262 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37247.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37247.log.gz 2026-02-19T21:12:55.263 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94104.log 2026-02-19T21:12:55.263 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55695.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55695.log.gz 2026-02-19T21:12:55.264 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127216.log 2026-02-19T21:12:55.264 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94104.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94104.log.gz 2026-02-19T21:12:55.264 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48570.log 2026-02-19T21:12:55.265 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127216.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127216.log.gz 2026-02-19T21:12:55.266 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115625.log 2026-02-19T21:12:55.267 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48570.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48570.log.gz 2026-02-19T21:12:55.267 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41198.log 2026-02-19T21:12:55.270 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115625.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115625.log.gz 2026-02-19T21:12:55.270 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126912.log 2026-02-19T21:12:55.271 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41198.log: 95.6% 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41198.log.gz 2026-02-19T21:12:55.271 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.80699.log.gz 2026-02-19T21:12:55.271 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60681.log 2026-02-19T21:12:55.272 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126912.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126912.log.gz 2026-02-19T21:12:55.273 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23443.log 2026-02-19T21:12:55.274 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60681.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112467.log 2026-02-19T21:12:55.274 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23443.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60681.log.gz 2026-02-19T21:12:55.277 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.87734.log 2026-02-19T21:12:55.278 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112467.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.23443.log.gz 2026-02-19T21:12:55.278 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43757.log 2026-02-19T21:12:55.286 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.3.87734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103129.log 2026-02-19T21:12:55.293 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43757.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43757.log.gz 2026-02-19T21:12:55.313 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113974.log 2026-02-19T21:12:55.322 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103129.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100187.log 2026-02-19T21:12:55.322 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113974.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113974.log.gz 2026-02-19T21:12:55.334 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74942.log 2026-02-19T21:12:55.334 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100187.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100187.log.gz 2026-02-19T21:12:55.334 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.103129.log.gz 2026-02-19T21:12:55.336 INFO:teuthology.orchestra.run.trial151.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.3.87734.log.gz 2026-02-19T21:12:55.337 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53824.log 2026-02-19T21:12:55.338 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74942.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74942.log.gz 2026-02-19T21:12:55.338 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51288.log 2026-02-19T21:12:55.339 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53824.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53824.log.gz 2026-02-19T21:12:55.339 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41295.log 2026-02-19T21:12:55.339 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51288.log: 77.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51288.log.gz 2026-02-19T21:12:55.340 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124321.log 2026-02-19T21:12:55.340 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41295.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41295.log.gz 2026-02-19T21:12:55.340 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124703.log 2026-02-19T21:12:55.341 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124321.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124321.log.gz 2026-02-19T21:12:55.341 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104580.log 2026-02-19T21:12:55.342 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124703.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124703.log.gz 2026-02-19T21:12:55.342 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28288.log 2026-02-19T21:12:55.343 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104580.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104580.log.gz 2026-02-19T21:12:55.343 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121195.log 2026-02-19T21:12:55.344 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28288.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28288.log.gz 2026-02-19T21:12:55.344 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100896.log 2026-02-19T21:12:55.345 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121195.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121195.log.gz 2026-02-19T21:12:55.345 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50948.log 2026-02-19T21:12:55.345 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100896.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100896.log.gz 2026-02-19T21:12:55.346 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26092.log 2026-02-19T21:12:55.346 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50948.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50948.log.gz 2026-02-19T21:12:55.346 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110352.log 2026-02-19T21:12:55.348 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26092.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26092.log.gz 2026-02-19T21:12:55.348 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88371.log 2026-02-19T21:12:55.348 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110352.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110352.log.gz 2026-02-19T21:12:55.349 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127278.log 2026-02-19T21:12:55.355 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88371.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84103.log.gz 2026-02-19T21:12:55.355 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77354.log 2026-02-19T21:12:55.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127278.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127278.log.gz 2026-02-19T21:12:55.356 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58896.log 2026-02-19T21:12:55.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77354.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77354.log.gz 2026-02-19T21:12:55.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94411.log 2026-02-19T21:12:55.358 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58896.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58896.log.gz 2026-02-19T21:12:55.358 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122254.log 2026-02-19T21:12:55.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94411.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94411.log.gz 2026-02-19T21:12:55.359 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103197.log 2026-02-19T21:12:55.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122254.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122254.log.gz 2026-02-19T21:12:55.360 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66289.log 2026-02-19T21:12:55.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103197.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103197.log.gz 2026-02-19T21:12:55.361 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82171.log 2026-02-19T21:12:55.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66289.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66289.log.gz 2026-02-19T21:12:55.362 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73522.log 2026-02-19T21:12:55.369 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82171.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42946.log 2026-02-19T21:12:55.382 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73522.log: 89.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63494.log 2026-02-19T21:12:55.382 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.73522.log.gz 2026-02-19T21:12:55.383 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42946.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42946.log.gz 2026-02-19T21:12:55.398 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19993.log 2026-02-19T21:12:55.398 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63494.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63494.log.gz 2026-02-19T21:12:55.400 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.106249.log.gz 2026-02-19T21:12:55.401 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48764.log 2026-02-19T21:12:55.402 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19993.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120477.log 2026-02-19T21:12:55.402 INFO:teuthology.orchestra.run.trial151.stderr: 80.6% -- replaced with /var/log/ceph/cluster2-client.admin.19993.log.gz 2026-02-19T21:12:55.403 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48764.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48764.log.gz 2026-02-19T21:12:55.403 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80213.log 2026-02-19T21:12:55.404 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120477.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120477.log.gz 2026-02-19T21:12:55.404 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105209.log 2026-02-19T21:12:55.417 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80213.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101552.log 2026-02-19T21:12:55.418 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105209.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105209.log.gz 2026-02-19T21:12:55.424 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.23508.log.gz 2026-02-19T21:12:55.429 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19351.log 2026-02-19T21:12:55.430 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101552.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101552.log.gz 2026-02-19T21:12:55.430 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21365.log 2026-02-19T21:12:55.431 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19351.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19351.log.gz 2026-02-19T21:12:55.431 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77206.log 2026-02-19T21:12:55.432 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21365.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127720.log 2026-02-19T21:12:55.441 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83308.log 2026-02-19T21:12:55.442 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77206.log: /var/log/ceph/cluster1-client.mirror.127720.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.127720.log.gz 2026-02-19T21:12:55.445 INFO:teuthology.orchestra.run.trial151.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77206.log.gz 2026-02-19T21:12:55.449 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109405.log 2026-02-19T21:12:55.450 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83308.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83308.log.gz 2026-02-19T21:12:55.461 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126268.log 2026-02-19T21:12:55.462 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109405.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.109405.log.gz 2026-02-19T21:12:55.464 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80974.log.gz 2026-02-19T21:12:55.465 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19301.log 2026-02-19T21:12:55.466 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126268.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126268.log.gz 2026-02-19T21:12:55.467 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62912.log 2026-02-19T21:12:55.468 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19301.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19301.log.gz 2026-02-19T21:12:55.468 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64817.log 2026-02-19T21:12:55.468 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62912.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62912.log.gz 2026-02-19T21:12:55.469 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124366.log 2026-02-19T21:12:55.470 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64817.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64817.log.gz 2026-02-19T21:12:55.470 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115536.log 2026-02-19T21:12:55.470 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124366.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124366.log.gz 2026-02-19T21:12:55.470 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70247.log 2026-02-19T21:12:55.471 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115536.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115536.log.gz 2026-02-19T21:12:55.472 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75217.log 2026-02-19T21:12:55.472 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70247.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70247.log.gz 2026-02-19T21:12:55.472 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40069.log 2026-02-19T21:12:55.473 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75217.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75217.log.gz 2026-02-19T21:12:55.473 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49544.log 2026-02-19T21:12:55.474 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40069.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40069.log.gz 2026-02-19T21:12:55.474 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69492.log 2026-02-19T21:12:55.475 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49544.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49544.log.gz 2026-02-19T21:12:55.475 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29394.log 2026-02-19T21:12:55.476 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69492.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69492.log.gz 2026-02-19T21:12:55.476 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60549.log 2026-02-19T21:12:55.477 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29394.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29394.log.gz 2026-02-19T21:12:55.477 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19873.log 2026-02-19T21:12:55.478 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60549.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60549.log.gz 2026-02-19T21:12:55.478 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85016.log 2026-02-19T21:12:55.479 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19873.log: 65.8% -- replaced with /var/log/ceph/cluster1-client.mirror.19873.log.gz 2026-02-19T21:12:55.479 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28711.log 2026-02-19T21:12:55.480 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.85016.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.85016.log.gz 2026-02-19T21:12:55.480 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121723.log 2026-02-19T21:12:55.480 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28711.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51233.log 2026-02-19T21:12:55.481 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.28711.log.gz 2026-02-19T21:12:55.481 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121723.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.121723.log.gz 2026-02-19T21:12:55.482 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122881.log 2026-02-19T21:12:55.483 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.51233.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.51233.log.gz 2026-02-19T21:12:55.483 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122094.log 2026-02-19T21:12:55.484 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122881.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.122881.log.gz 2026-02-19T21:12:55.484 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106778.log 2026-02-19T21:12:55.485 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122094.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.122094.log.gz 2026-02-19T21:12:55.485 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126953.log 2026-02-19T21:12:55.486 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.106778.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.106778.log.gz 2026-02-19T21:12:55.487 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79226.log 2026-02-19T21:12:55.487 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126953.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126953.log.gz 2026-02-19T21:12:55.488 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74783.log 2026-02-19T21:12:55.505 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79226.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67321.log 2026-02-19T21:12:55.506 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74783.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74783.log.gz 2026-02-19T21:12:55.517 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73875.log 2026-02-19T21:12:55.518 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67321.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67321.log.gz 2026-02-19T21:12:55.526 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54178.log.gz 2026-02-19T21:12:55.537 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114833.log 2026-02-19T21:12:55.538 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73875.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73875.log.gz 2026-02-19T21:12:55.539 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97662.log 2026-02-19T21:12:55.540 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114833.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114833.log.gz 2026-02-19T21:12:55.540 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35385.log 2026-02-19T21:12:55.541 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97662.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.97662.log.gz 2026-02-19T21:12:55.542 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68035.log 2026-02-19T21:12:55.543 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35385.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35385.log.gz 2026-02-19T21:12:55.543 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33736.log 2026-02-19T21:12:55.544 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68035.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68035.log.gz 2026-02-19T21:12:55.544 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23181.log 2026-02-19T21:12:55.545 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33736.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33736.log.gz 2026-02-19T21:12:55.545 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50805.log 2026-02-19T21:12:55.545 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23181.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51602.log 2026-02-19T21:12:55.546 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23181.log.gz 2026-02-19T21:12:55.548 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50805.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50805.log.gz 2026-02-19T21:12:55.548 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77501.log 2026-02-19T21:12:55.548 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.80869.log.gz 2026-02-19T21:12:55.548 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51602.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51602.log.gz 2026-02-19T21:12:55.548 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54150.log 2026-02-19T21:12:55.548 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77501.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19021.log 2026-02-19T21:12:55.552 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.54150.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77501.log.gz 2026-02-19T21:12:55.552 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.54150.log.gz 2026-02-19T21:12:55.553 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43951.log 2026-02-19T21:12:55.553 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43272.log 2026-02-19T21:12:55.553 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19021.log: /var/log/ceph/cluster2-client.mirror.43951.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19021.log.gz 2026-02-19T21:12:55.553 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120917.log 2026-02-19T21:12:55.553 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43272.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43951.log.gz 2026-02-19T21:12:55.553 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113198.log 2026-02-19T21:12:55.554 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120917.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43272.log.gz 2026-02-19T21:12:55.554 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.120917.log.gz 2026-02-19T21:12:55.554 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106560.log 2026-02-19T21:12:55.566 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113198.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.112467.log.gz 2026-02-19T21:12:55.566 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24055.log 2026-02-19T21:12:55.567 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106560.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28614.log 2026-02-19T21:12:55.567 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24055.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.106560.log.gz 2026-02-19T21:12:55.572 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24055.log.gz 2026-02-19T21:12:55.572 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101867.log 2026-02-19T21:12:55.572 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88371.log.gz 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112610.log 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.79825.log.gz 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101867.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28614.log.gz 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73584.log 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112610.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19401.log 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73584.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.112610.log.gz 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49703.log 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73584.log.gz 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19401.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127304.log 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101867.log.gz 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19401.log.gz 2026-02-19T21:12:55.573 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49703.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111411.log 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49703.log.gz 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52708.log 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127304.log: /var/log/ceph/cluster2-client.mirror.111411.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125437.log 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.52708.log: -- replaced with /var/log/ceph/cluster2-client.mirror.111411.log.gz 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83897.log 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52708.log.gz 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33286.log 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83897.log: 87.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95748.log 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.125437.log.gz 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98735.log 2026-02-19T21:12:55.574 INFO:teuthology.orchestra.run.trial151.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.95748.log: -- replaced with /var/log/ceph/cluster2-client.mirror.33286.log.gz 2026-02-19T21:12:55.575 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95748.log.gz 2026-02-19T21:12:55.586 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32960.log 2026-02-19T21:12:55.586 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98735.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66871.log 2026-02-19T21:12:55.587 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98735.log.gz 2026-02-19T21:12:55.587 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32960.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73972.log 2026-02-19T21:12:55.587 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66871.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32960.log.gz 2026-02-19T21:12:55.588 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66871.log.gz 2026-02-19T21:12:55.588 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34609.log 2026-02-19T21:12:55.589 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89372.log 2026-02-19T21:12:55.589 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73972.log: /var/log/ceph/cluster2-client.mirror.34609.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73972.log.gz 2026-02-19T21:12:55.589 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34609.log.gz 2026-02-19T21:12:55.589 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87305.log 2026-02-19T21:12:55.590 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89372.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89956.log 2026-02-19T21:12:55.590 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.89372.log.gz 2026-02-19T21:12:55.591 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87305.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89429.log 2026-02-19T21:12:55.591 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89956.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87305.log.gz 2026-02-19T21:12:55.591 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89956.log.gz 2026-02-19T21:12:55.591 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106059.log 2026-02-19T21:12:55.591 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121824.log 2026-02-19T21:12:55.591 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89429.log: /var/log/ceph/cluster2-client.mirror.106059.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.106059.log.gz 2026-02-19T21:12:55.593 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87953.log 2026-02-19T21:12:55.593 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121824.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121824.log.gz 2026-02-19T21:12:55.593 INFO:teuthology.orchestra.run.trial151.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89429.log.gz 2026-02-19T21:12:55.593 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117287.log 2026-02-19T21:12:55.593 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.87953.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98834.log 2026-02-19T21:12:55.594 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117287.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.87953.log.gz 2026-02-19T21:12:55.594 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73619.log 2026-02-19T21:12:55.601 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98834.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.84340.log 2026-02-19T21:12:55.602 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73619.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73619.log.gz 2026-02-19T21:12:55.609 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84644.log 2026-02-19T21:12:55.611 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.84340.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78483.log.gz 2026-02-19T21:12:55.612 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92760.log 2026-02-19T21:12:55.612 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.84644.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84644.log.gz 2026-02-19T21:12:55.612 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53255.log 2026-02-19T21:12:55.625 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92760.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91007.log 2026-02-19T21:12:55.626 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53255.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53255.log.gz 2026-02-19T21:12:55.641 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109972.log 2026-02-19T21:12:55.642 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.91007.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.91007.log.gz 2026-02-19T21:12:55.657 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23485.log 2026-02-19T21:12:55.658 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109972.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109972.log.gz 2026-02-19T21:12:55.665 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64981.log 2026-02-19T21:12:55.668 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23485.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45090.log.gz 2026-02-19T21:12:55.677 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43334.log 2026-02-19T21:12:55.678 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.64981.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64981.log.gz 2026-02-19T21:12:55.681 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.23873.log.gz 2026-02-19T21:12:55.684 INFO:teuthology.orchestra.run.trial151.stderr: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.23485.log.gz 2026-02-19T21:12:55.684 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78460.log 2026-02-19T21:12:55.686 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43334.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43334.log.gz 2026-02-19T21:12:55.686 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95794.log 2026-02-19T21:12:55.686 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78460.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78460.log.gz 2026-02-19T21:12:55.686 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124771.log 2026-02-19T21:12:55.688 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95794.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95794.log.gz 2026-02-19T21:12:55.688 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117220.log 2026-02-19T21:12:55.688 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124771.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124771.log.gz 2026-02-19T21:12:55.688 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124838.log 2026-02-19T21:12:55.689 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.117220.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.117220.log.gz 2026-02-19T21:12:55.689 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30879.log 2026-02-19T21:12:55.689 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124838.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124838.log.gz 2026-02-19T21:12:55.690 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77543.log 2026-02-19T21:12:55.690 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30879.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123106.log 2026-02-19T21:12:55.693 INFO:teuthology.orchestra.run.trial151.stderr: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30879.log.gz 2026-02-19T21:12:55.694 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77543.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77543.log.gz 2026-02-19T21:12:55.694 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126607.log 2026-02-19T21:12:55.695 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123106.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123106.log.gz 2026-02-19T21:12:55.696 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56603.log 2026-02-19T21:12:55.696 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126607.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126607.log.gz 2026-02-19T21:12:55.697 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76620.log 2026-02-19T21:12:55.697 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56603.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56603.log.gz 2026-02-19T21:12:55.697 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104208.log 2026-02-19T21:12:55.700 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76620.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76620.log.gz 2026-02-19T21:12:55.701 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54005.log 2026-02-19T21:12:55.701 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104208.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104208.log.gz 2026-02-19T21:12:55.702 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79893.log 2026-02-19T21:12:55.705 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54005.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54005.log.gz 2026-02-19T21:12:55.706 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34027.log 2026-02-19T21:12:55.707 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79893.log: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.79893.log.gz 2026-02-19T21:12:55.707 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56312.log 2026-02-19T21:12:55.707 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34027.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34027.log.gz 2026-02-19T21:12:55.707 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74069.log 2026-02-19T21:12:55.708 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56312.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56312.log.gz 2026-02-19T21:12:55.708 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116747.log 2026-02-19T21:12:55.709 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74069.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74069.log.gz 2026-02-19T21:12:55.710 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37297.log 2026-02-19T21:12:55.710 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116747.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.116747.log.gz 2026-02-19T21:12:55.710 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120139.log 2026-02-19T21:12:55.710 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37297.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75576.log 2026-02-19T21:12:55.711 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37297.log.gz 2026-02-19T21:12:55.711 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120139.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120139.log.gz 2026-02-19T21:12:55.712 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118367.log 2026-02-19T21:12:55.715 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.75576.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75576.log.gz 2026-02-19T21:12:55.715 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115143.log 2026-02-19T21:12:55.716 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.118367.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.118367.log.gz 2026-02-19T21:12:55.717 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24831.log 2026-02-19T21:12:55.717 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.115143.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.115143.log.gz 2026-02-19T21:12:55.718 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23463.log 2026-02-19T21:12:55.719 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24831.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.24831.log.gz 2026-02-19T21:12:55.719 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27512.log 2026-02-19T21:12:55.719 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.23463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24928.log 2026-02-19T21:12:55.723 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27512.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.23463.log.gz 2026-02-19T21:12:55.724 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71279.log 2026-02-19T21:12:55.724 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24928.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.27512.log.gz 2026-02-19T21:12:55.724 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24928.log.gz 2026-02-19T21:12:55.725 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.115625.log 2026-02-19T21:12:55.725 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71279.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71279.log.gz 2026-02-19T21:12:55.725 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18865.log 2026-02-19T21:12:55.738 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.115625.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117883.log 2026-02-19T21:12:55.738 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18865.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18865.log.gz 2026-02-19T21:12:55.750 INFO:teuthology.orchestra.run.trial151.stderr: 92.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75039.log 2026-02-19T21:12:55.751 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117883.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117883.log.gz 2026-02-19T21:12:55.751 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.0.115625.log.gz 2026-02-19T21:12:55.752 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108250.log 2026-02-19T21:12:55.752 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75039.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75039.log.gz 2026-02-19T21:12:55.753 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125407.log 2026-02-19T21:12:55.753 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108250.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108250.log.gz 2026-02-19T21:12:55.754 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62004.log 2026-02-19T21:12:55.754 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125407.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125407.log.gz 2026-02-19T21:12:55.755 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81275.log 2026-02-19T21:12:55.755 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62004.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62004.log.gz 2026-02-19T21:12:55.756 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127826.log 2026-02-19T21:12:55.757 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81275.log: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51148.log.gz 2026-02-19T21:12:55.758 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53546.log 2026-02-19T21:12:55.758 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.127826.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.127826.log.gz 2026-02-19T21:12:55.759 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120895.log 2026-02-19T21:12:55.759 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53546.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53546.log.gz 2026-02-19T21:12:55.760 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55633.log 2026-02-19T21:12:55.761 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120895.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.120895.log.gz 2026-02-19T21:12:55.761 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126006.log 2026-02-19T21:12:55.762 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55633.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55633.log.gz 2026-02-19T21:12:55.762 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104456.log 2026-02-19T21:12:55.763 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126006.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.126006.log.gz 2026-02-19T21:12:55.763 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110995.log 2026-02-19T21:12:55.763 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104456.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104456.log.gz 2026-02-19T21:12:55.764 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60937.log 2026-02-19T21:12:55.765 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110995.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.110995.log.gz 2026-02-19T21:12:55.765 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126153.log 2026-02-19T21:12:55.766 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25669.log 2026-02-19T21:12:55.767 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126153.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126153.log.gz 2026-02-19T21:12:55.768 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119549.log 2026-02-19T21:12:55.768 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25669.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25669.log.gz 2026-02-19T21:12:55.768 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63203.log 2026-02-19T21:12:55.770 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119549.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.119549.log.gz 2026-02-19T21:12:55.770 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60937.log.gz 2026-02-19T21:12:55.770 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61457.log 2026-02-19T21:12:55.771 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63203.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63203.log.gz 2026-02-19T21:12:55.771 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34415.log 2026-02-19T21:12:55.772 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61457.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61457.log.gz 2026-02-19T21:12:55.772 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103107.log 2026-02-19T21:12:55.773 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34415.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34415.log.gz 2026-02-19T21:12:55.773 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71958.log 2026-02-19T21:12:55.774 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.103107.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.103107.log.gz 2026-02-19T21:12:55.774 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106961.log 2026-02-19T21:12:55.775 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71958.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71958.log.gz 2026-02-19T21:12:55.775 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105469.log 2026-02-19T21:12:55.776 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106961.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106961.log.gz 2026-02-19T21:12:55.776 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123263.log 2026-02-19T21:12:55.777 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.105469.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.105469.log.gz 2026-02-19T21:12:55.777 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122183.log 2026-02-19T21:12:55.778 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123263.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123263.log.gz 2026-02-19T21:12:55.778 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36681.log 2026-02-19T21:12:55.779 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122183.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.122183.log.gz 2026-02-19T21:12:55.779 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105895.log 2026-02-19T21:12:55.780 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36681.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36681.log.gz 2026-02-19T21:12:55.780 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64076.log 2026-02-19T21:12:55.781 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105895.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105895.log.gz 2026-02-19T21:12:55.781 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73425.log 2026-02-19T21:12:55.782 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64076.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64076.log.gz 2026-02-19T21:12:55.782 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18638.log 2026-02-19T21:12:55.783 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73425.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73425.log.gz 2026-02-19T21:12:55.783 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41392.log 2026-02-19T21:12:55.784 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18638.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18638.log.gz 2026-02-19T21:12:55.784 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124186.log 2026-02-19T21:12:55.785 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41392.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41392.log.gz 2026-02-19T21:12:55.785 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109616.log 2026-02-19T21:12:55.786 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124186.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124186.log.gz 2026-02-19T21:12:55.787 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70891.log 2026-02-19T21:12:55.787 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79756.log.gz 2026-02-19T21:12:55.787 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109616.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109616.log.gz 2026-02-19T21:12:55.787 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39642.log 2026-02-19T21:12:55.789 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62489.log 2026-02-19T21:12:55.789 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70891.log: /var/log/ceph/cluster2-client.mirror.39642.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70891.log.gz 2026-02-19T21:12:55.790 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39642.log.gz 2026-02-19T21:12:55.790 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117347.log 2026-02-19T21:12:55.790 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62489.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62489.log.gz 2026-02-19T21:12:55.790 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75790.log 2026-02-19T21:12:55.790 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92545.log 2026-02-19T21:12:55.790 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117347.log: /var/log/ceph/cluster2-client.mirror.75790.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75790.log.gz 2026-02-19T21:12:55.790 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117347.log.gz 2026-02-19T21:12:55.792 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22286.log 2026-02-19T21:12:55.792 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18943.log 2026-02-19T21:12:55.792 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92545.log: /var/log/ceph/cluster2-client.mirror.22286.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92545.log.gz 2026-02-19T21:12:55.792 INFO:teuthology.orchestra.run.trial151.stderr: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47856.log 2026-02-19T21:12:55.793 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.22286.log.gz 2026-02-19T21:12:55.793 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18943.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18943.log.gz 2026-02-19T21:12:55.793 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86318.log 2026-02-19T21:12:55.793 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112348.log 2026-02-19T21:12:55.793 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47856.log: /var/log/ceph/cluster2-client.mirror.86318.log: 88.8% 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86318.log.gz 2026-02-19T21:12:55.793 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.47856.log.gz 2026-02-19T21:12:55.793 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63556.log 2026-02-19T21:12:55.795 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.112348.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.112348.log.gz 2026-02-19T21:12:55.795 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36355.log 2026-02-19T21:12:55.795 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78289.log 2026-02-19T21:12:55.795 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63556.log: /var/log/ceph/cluster2-client.mirror.36355.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63556.log.gz 2026-02-19T21:12:55.795 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36355.log.gz 2026-02-19T21:12:55.795 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104977.log 2026-02-19T21:12:55.795 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22751.log 2026-02-19T21:12:55.796 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78289.log: /var/log/ceph/cluster2-client.mirror.104977.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104977.log.gz 2026-02-19T21:12:55.805 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120264.log 2026-02-19T21:12:55.806 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22751.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.22751.log.gz 2026-02-19T21:12:55.806 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72796.log 2026-02-19T21:12:55.807 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120264.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.120264.log.gz 2026-02-19T21:12:55.807 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112846.log 2026-02-19T21:12:55.810 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.72796.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.21365.log.gz 2026-02-19T21:12:55.813 INFO:teuthology.orchestra.run.trial151.stderr: 93.6%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23418.log 2026-02-19T21:12:55.813 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.72796.log.gz 2026-02-19T21:12:55.813 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112846.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114995.log 2026-02-19T21:12:55.813 INFO:teuthology.orchestra.run.trial151.stderr: 88.3%/var/log/ceph/cluster1-client.mirror.23418.log: -- replaced with /var/log/ceph/cluster2-client.mirror.112846.log.gz 2026-02-19T21:12:55.813 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55148.log 2026-02-19T21:12:55.813 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61519.log 2026-02-19T21:12:55.814 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.23418.log.gz 2026-02-19T21:12:55.814 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55148.log: /var/log/ceph/cluster2-client.mirror.114995.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55148.log.gz 2026-02-19T21:12:55.814 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114995.log.gz 2026-02-19T21:12:55.814 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76428.log 2026-02-19T21:12:55.814 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61519.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109046.log 2026-02-19T21:12:55.815 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76428.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61519.log.gz 2026-02-19T21:12:55.815 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127399.log 2026-02-19T21:12:55.817 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109046.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76428.log.gz 2026-02-19T21:12:55.817 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93006.log 2026-02-19T21:12:55.818 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127399.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.127399.log.gz 2026-02-19T21:12:55.819 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124028.log 2026-02-19T21:12:55.819 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93006.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.93006.log.gz 2026-02-19T21:12:55.820 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96545.log 2026-02-19T21:12:55.820 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124028.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124028.log.gz 2026-02-19T21:12:55.820 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107911.log 2026-02-19T21:12:55.822 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96545.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96545.log.gz 2026-02-19T21:12:55.822 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65026.log 2026-02-19T21:12:55.822 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107911.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107911.log.gz 2026-02-19T21:12:55.822 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28385.log 2026-02-19T21:12:55.826 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.65026.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65026.log.gz 2026-02-19T21:12:55.826 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126847.log 2026-02-19T21:12:55.827 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28385.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28385.log.gz 2026-02-19T21:12:55.827 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40263.log 2026-02-19T21:12:55.828 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126847.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126847.log.gz 2026-02-19T21:12:55.828 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53061.log 2026-02-19T21:12:55.829 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40263.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40263.log.gz 2026-02-19T21:12:55.829 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76686.log 2026-02-19T21:12:55.829 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53061.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53061.log.gz 2026-02-19T21:12:55.829 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98235.log 2026-02-19T21:12:55.833 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76686.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76686.log.gz 2026-02-19T21:12:55.834 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38575.log 2026-02-19T21:12:55.834 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98235.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98235.log.gz 2026-02-19T21:12:55.834 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116983.log 2026-02-19T21:12:55.835 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38575.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38575.log.gz 2026-02-19T21:12:55.835 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19201.log 2026-02-19T21:12:55.836 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116983.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116983.log.gz 2026-02-19T21:12:55.836 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119287.log 2026-02-19T21:12:55.836 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.19201.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19201.log.gz 2026-02-19T21:12:55.836 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15940.log 2026-02-19T21:12:55.838 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119287.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119287.log.gz 2026-02-19T21:12:55.839 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100973.log 2026-02-19T21:12:55.839 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.15940.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15940.log.gz 2026-02-19T21:12:55.839 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45534.log 2026-02-19T21:12:55.840 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100973.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100973.log.gz 2026-02-19T21:12:55.840 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126025.log 2026-02-19T21:12:55.840 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45534.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45534.log.gz 2026-02-19T21:12:55.841 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95037.log 2026-02-19T21:12:55.841 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126025.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.126025.log.gz 2026-02-19T21:12:55.842 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97185.log 2026-02-19T21:12:55.844 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95037.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.95037.log.gz 2026-02-19T21:12:55.844 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101629.log 2026-02-19T21:12:55.845 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97185.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97185.log.gz 2026-02-19T21:12:55.845 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64623.log 2026-02-19T21:12:55.846 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101629.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101629.log.gz 2026-02-19T21:12:55.846 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118605.log 2026-02-19T21:12:55.847 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64623.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64623.log.gz 2026-02-19T21:12:55.847 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91327.log 2026-02-19T21:12:55.847 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.118605.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.118605.log.gz 2026-02-19T21:12:55.848 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119174.log 2026-02-19T21:12:55.849 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.91327.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.91327.log.gz 2026-02-19T21:12:55.850 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55598.log 2026-02-19T21:12:55.850 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119174.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.119174.log.gz 2026-02-19T21:12:55.851 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71702.log 2026-02-19T21:12:55.851 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55598.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55598.log.gz 2026-02-19T21:12:55.851 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58217.log 2026-02-19T21:12:55.852 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71702.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71702.log.gz 2026-02-19T21:12:55.853 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91307.log 2026-02-19T21:12:55.853 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58217.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58217.log.gz 2026-02-19T21:12:55.853 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96380.log 2026-02-19T21:12:55.855 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.91307.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.91307.log.gz 2026-02-19T21:12:55.855 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29553.log 2026-02-19T21:12:55.856 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.96380.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.96380.log.gz 2026-02-19T21:12:55.857 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107539.log 2026-02-19T21:12:55.857 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29553.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29553.log.gz 2026-02-19T21:12:55.857 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30491.log 2026-02-19T21:12:55.877 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107539.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29456.log 2026-02-19T21:12:55.878 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30491.log: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.3.78978.log.gz 2026-02-19T21:12:55.878 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30491.log.gz 2026-02-19T21:12:55.879 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34997.log 2026-02-19T21:12:55.879 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29456.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29456.log.gz 2026-02-19T21:12:55.879 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45666.log 2026-02-19T21:12:55.880 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34997.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34997.log.gz 2026-02-19T21:12:55.880 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98506.log 2026-02-19T21:12:55.881 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45666.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45666.log.gz 2026-02-19T21:12:55.881 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52219.log 2026-02-19T21:12:55.882 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98506.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98506.log.gz 2026-02-19T21:12:55.883 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57573.log 2026-02-19T21:12:55.883 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52219.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52219.log.gz 2026-02-19T21:12:55.883 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73002.log 2026-02-19T21:12:55.884 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57573.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57573.log.gz 2026-02-19T21:12:55.884 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97451.log 2026-02-19T21:12:55.885 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73002.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73002.log.gz 2026-02-19T21:12:55.885 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125502.log 2026-02-19T21:12:55.886 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97451.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97451.log.gz 2026-02-19T21:12:55.886 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77950.log 2026-02-19T21:12:55.886 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125502.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125502.log.gz 2026-02-19T21:12:55.887 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66060.log 2026-02-19T21:12:55.893 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77950.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46407.log 2026-02-19T21:12:55.906 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54047.log 2026-02-19T21:12:55.906 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46407.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46407.log.gz 2026-02-19T21:12:55.909 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66060.log.gz 2026-02-19T21:12:55.921 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95976.log 2026-02-19T21:12:55.926 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54047.log: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54047.log.gz 2026-02-19T21:12:55.933 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119500.log 2026-02-19T21:12:55.945 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95976.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91053.log 2026-02-19T21:12:55.946 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119500.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.119500.log.gz 2026-02-19T21:12:55.954 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65114.log.gz 2026-02-19T21:12:55.957 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.113198.log.gz 2026-02-19T21:12:55.957 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49020.log 2026-02-19T21:12:55.958 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.91053.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.91053.log.gz 2026-02-19T21:12:55.969 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48147.log 2026-02-19T21:12:55.970 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49020.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78879.log.gz 2026-02-19T21:12:55.970 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49020.log.gz 2026-02-19T21:12:55.971 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23814.log 2026-02-19T21:12:55.972 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48147.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32578.log 2026-02-19T21:12:55.972 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48147.log.gz 2026-02-19T21:12:55.972 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.23814.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79689.log 2026-02-19T21:12:55.973 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32578.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.23814.log.gz 2026-02-19T21:12:55.973 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32578.log.gz 2026-02-19T21:12:55.973 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58543.log 2026-02-19T21:12:55.973 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79689.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121493.log 2026-02-19T21:12:55.974 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58543.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58543.log.gz 2026-02-19T21:12:55.974 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.125637.log 2026-02-19T21:12:55.975 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121493.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.121493.log.gz 2026-02-19T21:12:55.976 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92820.log 2026-02-19T21:12:55.980 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.1.125637.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.1.125637.log.gz 2026-02-19T21:12:55.980 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114585.log 2026-02-19T21:12:55.980 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92820.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92820.log.gz 2026-02-19T21:12:55.981 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54240.log 2026-02-19T21:12:55.981 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114585.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114585.log.gz 2026-02-19T21:12:55.982 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97942.log 2026-02-19T21:12:55.983 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54240.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54240.log.gz 2026-02-19T21:12:55.983 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127888.log 2026-02-19T21:12:55.983 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97942.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.97942.log.gz 2026-02-19T21:12:55.984 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32419.log 2026-02-19T21:12:55.985 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127888.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.127888.log.gz 2026-02-19T21:12:55.986 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50902.log 2026-02-19T21:12:55.986 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32419.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.32419.log.gz 2026-02-19T21:12:55.986 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103740.log 2026-02-19T21:12:55.988 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50902.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50902.log.gz 2026-02-19T21:12:55.988 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127048.log 2026-02-19T21:12:55.988 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.103740.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.103740.log.gz 2026-02-19T21:12:55.989 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71605.log 2026-02-19T21:12:55.990 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.127048.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.127048.log.gz 2026-02-19T21:12:55.990 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95149.log 2026-02-19T21:12:55.991 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71605.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71605.log.gz 2026-02-19T21:12:55.991 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20027.log 2026-02-19T21:12:55.991 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95149.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95149.log.gz 2026-02-19T21:12:55.992 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112711.log 2026-02-19T21:12:55.992 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.20027.log: 73.2% -- replaced with /var/log/ceph/cluster2-client.admin.20027.log.gz 2026-02-19T21:12:55.992 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123038.log 2026-02-19T21:12:55.993 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112711.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112711.log.gz 2026-02-19T21:12:55.993 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25801.log 2026-02-19T21:12:55.995 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123038.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123038.log.gz 2026-02-19T21:12:55.995 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41524.log 2026-02-19T21:12:55.995 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25801.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25801.log.gz 2026-02-19T21:12:55.995 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103426.log 2026-02-19T21:12:55.996 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41524.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41524.log.gz 2026-02-19T21:12:55.996 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59804.log 2026-02-19T21:12:55.997 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103426.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103426.log.gz 2026-02-19T21:12:55.997 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56956.log 2026-02-19T21:12:55.997 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59804.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59804.log.gz 2026-02-19T21:12:55.998 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53789.log 2026-02-19T21:12:55.998 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79426.log.gz 2026-02-19T21:12:55.998 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56956.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118983.log 2026-02-19T21:12:55.998 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53789.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56956.log.gz 2026-02-19T21:12:55.999 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38513.log 2026-02-19T21:12:55.999 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53789.log.gz 2026-02-19T21:12:55.999 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32995.log 2026-02-19T21:12:56.001 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118983.log.gz/var/log/ceph/cluster2-client.mirror.38513.log: 2026-02-19T21:12:56.001 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.98834.log.gz 2026-02-19T21:12:56.001 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21879.log 2026-02-19T21:12:56.001 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38513.log.gz 2026-02-19T21:12:56.001 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61228.log 2026-02-19T21:12:56.001 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21879.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120690.log 2026-02-19T21:12:56.001 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32995.log.gz 2026-02-19T21:12:56.003 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61228.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21879.log.gz 2026-02-19T21:12:56.003 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61228.log.gz 2026-02-19T21:12:56.003 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125460.log 2026-02-19T21:12:56.003 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120690.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74686.log 2026-02-19T21:12:56.003 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125460.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.120690.log.gz 2026-02-19T21:12:56.003 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.125460.log.gz 2026-02-19T21:12:56.003 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125112.log 2026-02-19T21:12:56.004 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74686.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28909.log 2026-02-19T21:12:56.004 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125112.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74686.log.gz 2026-02-19T21:12:56.004 INFO:teuthology.orchestra.run.trial151.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.125112.log.gz 2026-02-19T21:12:56.005 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77163.log 2026-02-19T21:12:56.005 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28909.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86679.log 2026-02-19T21:12:56.005 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77163.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113447.log 2026-02-19T21:12:56.005 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28909.log.gz 2026-02-19T21:12:56.006 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94216.log 2026-02-19T21:12:56.007 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86679.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.86679.log.gz 2026-02-19T21:12:56.007 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113447.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113447.log.gz 2026-02-19T21:12:56.007 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51103.log 2026-02-19T21:12:56.007 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.94216.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.94216.log.gz 2026-02-19T21:12:56.008 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88349.log 2026-02-19T21:12:56.008 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77163.log.gz 2026-02-19T21:12:56.011 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.51103.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.51103.log.gz 2026-02-19T21:12:56.012 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37840.log 2026-02-19T21:12:56.012 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.88349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48182.log 2026-02-19T21:12:56.012 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37840.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89094.log 2026-02-19T21:12:56.013 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48182.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37840.log.gz 2026-02-19T21:12:56.013 INFO:teuthology.orchestra.run.trial151.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.88349.log.gz 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48182.log.gz 2026-02-19T21:12:56.014 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:56.014 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78236.log 2026-02-19T21:12:56.014 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89094.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89094.log.gz 2026-02-19T21:12:56.014 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127668.log 2026-02-19T21:12:56.015 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114949.log 2026-02-19T21:12:56.015 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127668.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76165.log 2026-02-19T21:12:56.015 INFO:teuthology.orchestra.run.trial151.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127668.log.gz 2026-02-19T21:12:56.016 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114949.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114949.log.gz 2026-02-19T21:12:56.017 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26965.log 2026-02-19T21:12:56.018 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93126.log 2026-02-19T21:12:56.018 INFO:teuthology.orchestra.run.trial151.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76165.log.gz 2026-02-19T21:12:56.019 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26965.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82545.log 2026-02-19T21:12:56.019 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93126.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26965.log.gz 2026-02-19T21:12:56.019 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93126.log.gz 2026-02-19T21:12:56.020 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128891.log 2026-02-19T21:12:56.020 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31158.log 2026-02-19T21:12:56.020 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82545.log: /var/log/ceph/cluster2-client.admin.128891.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128891.log.gz 2026-02-19T21:12:56.020 INFO:teuthology.orchestra.run.trial151.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82545.log.gz 2026-02-19T21:12:56.021 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.19479.log 2026-02-19T21:12:56.021 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31158.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36875.log 2026-02-19T21:12:56.021 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.3.19479.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31158.log.gz 2026-02-19T21:12:56.022 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16053.log 2026-02-19T21:12:56.022 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36875.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36875.log.gz 2026-02-19T21:12:56.022 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128580.log 2026-02-19T21:12:56.022 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.16053.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16053.log.gz 2026-02-19T21:12:56.023 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102555.log 2026-02-19T21:12:56.023 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128580.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128580.log.gz 2026-02-19T21:12:56.024 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121173.log 2026-02-19T21:12:56.025 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102555.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.3.19479.log.gz 2026-02-19T21:12:56.029 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22065.log 2026-02-19T21:12:56.030 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121173.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121173.log.gz 2026-02-19T21:12:56.030 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28874.log 2026-02-19T21:12:56.031 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22065.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22065.log.gz 2026-02-19T21:12:56.031 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107166.log 2026-02-19T21:12:56.032 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28874.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28874.log.gz 2026-02-19T21:12:56.032 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92738.log 2026-02-19T21:12:56.033 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107166.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107166.log.gz 2026-02-19T21:12:56.033 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94765.log 2026-02-19T21:12:56.035 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.92738.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.92738.log.gz 2026-02-19T21:12:56.035 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87841.log 2026-02-19T21:12:56.036 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94765.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94765.log.gz 2026-02-19T21:12:56.036 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115030.log 2026-02-19T21:12:56.037 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87841.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87841.log.gz 2026-02-19T21:12:56.037 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.46686.log 2026-02-19T21:12:56.038 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115030.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115030.log.gz 2026-02-19T21:12:56.038 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99187.log 2026-02-19T21:12:56.040 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.46686.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.46686.log.gz 2026-02-19T21:12:56.040 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23088.log 2026-02-19T21:12:56.041 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99187.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99187.log.gz 2026-02-19T21:12:56.041 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49800.log 2026-02-19T21:12:56.042 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23088.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.20874.log.gz 2026-02-19T21:12:56.042 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.23088.log.gz 2026-02-19T21:12:56.042 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121895.log 2026-02-19T21:12:56.043 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123646.log 2026-02-19T21:12:56.043 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49800.log: /var/log/ceph/cluster2-client.mirror.121895.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49800.log.gz 2026-02-19T21:12:56.044 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121895.log.gz 2026-02-19T21:12:56.044 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102906.log 2026-02-19T21:12:56.044 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43431.log 2026-02-19T21:12:56.045 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123646.log: /var/log/ceph/cluster2-client.mirror.102906.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123646.log.gz 2026-02-19T21:12:56.045 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102906.log.gz 2026-02-19T21:12:56.045 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26057.log 2026-02-19T21:12:56.046 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120872.log 2026-02-19T21:12:56.046 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43431.log: /var/log/ceph/cluster2-client.mirror.26057.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43431.log.gz 2026-02-19T21:12:56.046 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26057.log.gz 2026-02-19T21:12:56.046 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19689.log 2026-02-19T21:12:56.046 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74977.log 2026-02-19T21:12:56.047 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120872.log: /var/log/ceph/cluster1-client.admin.19689.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19689.log.gz 2026-02-19T21:12:56.047 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32773.log 2026-02-19T21:12:56.048 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74977.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74977.log.gz 2026-02-19T21:12:56.048 INFO:teuthology.orchestra.run.trial151.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.120872.log.gz 2026-02-19T21:12:56.049 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53193.log 2026-02-19T21:12:56.050 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32773.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87216.log 2026-02-19T21:12:56.050 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53193.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32773.log.gz 2026-02-19T21:12:56.050 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53193.log.gz 2026-02-19T21:12:56.051 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113832.log 2026-02-19T21:12:56.051 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87216.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41974.log 2026-02-19T21:12:56.051 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.87216.log.gz 2026-02-19T21:12:56.051 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.113832.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118676.log 2026-02-19T21:12:56.052 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41974.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41974.log.gz 2026-02-19T21:12:56.052 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112179.log 2026-02-19T21:12:56.053 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118676.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.113832.log.gz 2026-02-19T21:12:56.053 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.118676.log.gz 2026-02-19T21:12:56.053 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78334.log 2026-02-19T21:12:56.053 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37551.log 2026-02-19T21:12:56.054 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112179.log: /var/log/ceph/cluster2-client.mirror.78334.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112179.log.gz 2026-02-19T21:12:56.054 INFO:teuthology.orchestra.run.trial151.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78334.log.gz 2026-02-19T21:12:56.055 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71182.log 2026-02-19T21:12:56.055 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49117.log 2026-02-19T21:12:56.055 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37551.log: /var/log/ceph/cluster2-client.mirror.71182.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122536.log 2026-02-19T21:12:56.055 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71182.log.gz 2026-02-19T21:12:56.056 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118573.log 2026-02-19T21:12:56.056 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122536.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.122536.log.gz 2026-02-19T21:12:56.057 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72187.log 2026-02-19T21:12:56.057 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118573.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118573.log.gz 2026-02-19T21:12:56.058 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65415.log 2026-02-19T21:12:56.058 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72187.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72187.log.gz 2026-02-19T21:12:56.059 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121451.log 2026-02-19T21:12:56.059 INFO:teuthology.orchestra.run.trial151.stderr: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.37551.log.gz 2026-02-19T21:12:56.059 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65415.log: /var/log/ceph/cluster2-client.mirror.49117.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49117.log.gz 2026-02-19T21:12:56.060 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19478.log 2026-02-19T21:12:56.060 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121451.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.121451.log.gz 2026-02-19T21:12:56.061 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123466.log 2026-02-19T21:12:56.061 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.19478.log: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.19478.log.gz 2026-02-19T21:12:56.061 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101092.log 2026-02-19T21:12:56.062 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123466.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123466.log.gz 2026-02-19T21:12:56.062 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101594.log 2026-02-19T21:12:56.063 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101092.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101092.log.gz 2026-02-19T21:12:56.063 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78918.log 2026-02-19T21:12:56.063 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101594.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101594.log.gz 2026-02-19T21:12:56.064 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83331.log 2026-02-19T21:12:56.064 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78918.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78918.log.gz 2026-02-19T21:12:56.065 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117483.log 2026-02-19T21:12:56.071 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83331.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83897.log.gz 2026-02-19T21:12:56.073 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70794.log 2026-02-19T21:12:56.074 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117483.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.117483.log.gz 2026-02-19T21:12:56.091 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.107539.log.gz 2026-02-19T21:12:56.093 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122347.log 2026-02-19T21:12:56.093 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70794.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69407.log 2026-02-19T21:12:56.094 INFO:teuthology.orchestra.run.trial151.stderr: 89.2%/var/log/ceph/cluster2-client.mirror.122347.log: -- replaced with /var/log/ceph/cluster2-client.mirror.70794.log.gz 2026-02-19T21:12:56.094 INFO:teuthology.orchestra.run.trial151.stderr: 88.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123218.log 2026-02-19T21:12:56.094 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.122347.log.gz 2026-02-19T21:12:56.095 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69407.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69407.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119662.log 2026-02-19T21:12:56.095 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:56.095 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123218.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77694.log 2026-02-19T21:12:56.096 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119662.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123218.log.gz 2026-02-19T21:12:56.096 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39095.log 2026-02-19T21:12:56.096 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77694.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119662.log.gz 2026-02-19T21:12:56.097 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85543.log 2026-02-19T21:12:56.098 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39095.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39095.log.gz 2026-02-19T21:12:56.098 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114435.log 2026-02-19T21:12:56.098 INFO:teuthology.orchestra.run.trial151.stderr: 91.4%/var/log/ceph/cluster2-client.mirror.85543.log: -- replaced with /var/log/ceph/cluster2-client.mirror.77694.log.gz 2026-02-19T21:12:56.098 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85543.log.gz 2026-02-19T21:12:56.099 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77312.log 2026-02-19T21:12:56.099 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77249.log 2026-02-19T21:12:56.099 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114435.log: /var/log/ceph/cluster2-client.mirror.77312.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114435.log.gz 2026-02-19T21:12:56.100 INFO:teuthology.orchestra.run.trial151.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77312.log.gz 2026-02-19T21:12:56.100 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70053.log 2026-02-19T21:12:56.100 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64720.log 2026-02-19T21:12:56.100 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77249.log: /var/log/ceph/cluster2-client.mirror.70053.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.70053.log.gz 2026-02-19T21:12:56.101 INFO:teuthology.orchestra.run.trial151.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77249.log.gz 2026-02-19T21:12:56.101 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26736.log 2026-02-19T21:12:56.101 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64720.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42494.log 2026-02-19T21:12:56.101 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64720.log.gz 2026-02-19T21:12:56.102 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104679.log 2026-02-19T21:12:56.102 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26736.log: /var/log/ceph/cluster2-client.mirror.42494.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26736.log.gz 2026-02-19T21:12:56.103 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42494.log.gz 2026-02-19T21:12:56.103 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78941.log 2026-02-19T21:12:56.103 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109359.log 2026-02-19T21:12:56.117 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78941.log: /var/log/ceph/cluster2-client.mirror.104679.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96731.log 2026-02-19T21:12:56.118 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109359.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109359.log.gz 2026-02-19T21:12:56.123 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.117287.log.gz 2026-02-19T21:12:56.124 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26445.log 2026-02-19T21:12:56.124 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96731.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96731.log.gz 2026-02-19T21:12:56.125 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85655.log 2026-02-19T21:12:56.125 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26445.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26445.log.gz 2026-02-19T21:12:56.126 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125727.log 2026-02-19T21:12:56.126 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85655.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85655.log.gz 2026-02-19T21:12:56.126 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122746.log 2026-02-19T21:12:56.127 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125727.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.125727.log.gz 2026-02-19T21:12:56.127 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20202.log 2026-02-19T21:12:56.128 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122746.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.122746.log.gz 2026-02-19T21:12:56.128 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60390.log 2026-02-19T21:12:56.129 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20202.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.20202.log.gz 2026-02-19T21:12:56.129 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45340.log 2026-02-19T21:12:56.129 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60390.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60390.log.gz 2026-02-19T21:12:56.130 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77781.log 2026-02-19T21:12:56.130 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45340.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45340.log.gz 2026-02-19T21:12:56.131 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102336.log 2026-02-19T21:12:56.131 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.77781.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.77781.log.gz 2026-02-19T21:12:56.131 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66774.log 2026-02-19T21:12:56.132 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102336.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102336.log.gz 2026-02-19T21:12:56.132 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65866.log 2026-02-19T21:12:56.133 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66774.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66774.log.gz 2026-02-19T21:12:56.133 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18709.log 2026-02-19T21:12:56.134 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65866.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65866.log.gz 2026-02-19T21:12:56.134 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121150.log 2026-02-19T21:12:56.134 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18709.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18709.log.gz 2026-02-19T21:12:56.135 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109324.log 2026-02-19T21:12:56.135 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121150.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81275.log.gz 2026-02-19T21:12:56.135 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53581.log 2026-02-19T21:12:56.136 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109324.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121150.log.gz 2026-02-19T21:12:56.136 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109324.log.gz 2026-02-19T21:12:56.136 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69559.log 2026-02-19T21:12:56.137 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53581.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105636.log 2026-02-19T21:12:56.137 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.69559.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53581.log.gz 2026-02-19T21:12:56.138 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46886.log 2026-02-19T21:12:56.138 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69559.log.gz 2026-02-19T21:12:56.138 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75968.log 2026-02-19T21:12:56.138 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105636.log.gz 2026-02-19T21:12:56.140 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46886.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115076.log 2026-02-19T21:12:56.140 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46886.log.gz 2026-02-19T21:12:56.140 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75968.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22937.log 2026-02-19T21:12:56.140 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75968.log.gz 2026-02-19T21:12:56.140 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115076.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115076.log.gz 2026-02-19T21:12:56.140 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19856.log 2026-02-19T21:12:56.141 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119642.log 2026-02-19T21:12:56.141 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19856.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22937.log.gz 2026-02-19T21:12:56.141 INFO:teuthology.orchestra.run.trial151.stderr: 68.6% -- replaced with /var/log/ceph/cluster1-client.mirror.19856.log.gz 2026-02-19T21:12:56.141 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113720.log 2026-02-19T21:12:56.141 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119642.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103062.log 2026-02-19T21:12:56.142 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113720.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119642.log.gz 2026-02-19T21:12:56.142 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.113720.log.gz 2026-02-19T21:12:56.142 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126891.log 2026-02-19T21:12:56.142 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.103062.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.103062.log.gz 2026-02-19T21:12:56.143 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64914.log 2026-02-19T21:12:56.143 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126891.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126891.log.gz 2026-02-19T21:12:56.143 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127175.log 2026-02-19T21:12:56.144 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64914.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64914.log.gz 2026-02-19T21:12:56.144 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40457.log 2026-02-19T21:12:56.145 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127175.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127175.log.gz 2026-02-19T21:12:56.145 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.109046.log.gz 2026-02-19T21:12:56.145 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96981.log 2026-02-19T21:12:56.147 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40457.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57247.log 2026-02-19T21:12:56.147 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.96981.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127606.log 2026-02-19T21:12:56.147 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40457.log.gz 2026-02-19T21:12:56.147 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57247.log: 88.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111741.log 2026-02-19T21:12:56.147 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.96981.log.gz 2026-02-19T21:12:56.147 INFO:teuthology.orchestra.run.trial151.stderr: 88.9%/var/log/ceph/cluster2-client.mirror.127606.log: -- replaced with /var/log/ceph/cluster2-client.mirror.57247.log.gz 2026-02-19T21:12:56.147 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128178.log 2026-02-19T21:12:56.148 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111741.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115347.log 2026-02-19T21:12:56.148 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.128178.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.128178.log.gz 2026-02-19T21:12:56.148 INFO:teuthology.orchestra.run.trial151.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.127606.log.gz 2026-02-19T21:12:56.148 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80213.log.gz 2026-02-19T21:12:56.149 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88730.log 2026-02-19T21:12:56.150 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115347.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55307.log 2026-02-19T21:12:56.150 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88730.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115347.log.gz 2026-02-19T21:12:56.150 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128005.log 2026-02-19T21:12:56.150 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55307.log: gzip -5 --verbose 88.3% -- /var/log/ceph/cluster2-client.mirror.80190.log -- replaced with /var/log/ceph/cluster2-client.mirror.88730.log.gz 2026-02-19T21:12:56.150 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:56.151 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.128005.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55307.log.gz 2026-02-19T21:12:56.151 INFO:teuthology.orchestra.run.trial151.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128005.log.gz 2026-02-19T21:12:56.152 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124073.log 2026-02-19T21:12:56.152 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80190.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62586.log 2026-02-19T21:12:56.152 INFO:teuthology.orchestra.run.trial151.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80190.log.gz/var/log/ceph/cluster1-client.mirror.124073.log: 2026-02-19T21:12:56.152 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31802.log 2026-02-19T21:12:56.152 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124073.log.gz 2026-02-19T21:12:56.152 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62586.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62586.log.gz 2026-02-19T21:12:56.153 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70212.log 2026-02-19T21:12:56.153 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31802.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67903.log 2026-02-19T21:12:56.153 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70212.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31802.log.gz 2026-02-19T21:12:56.153 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70212.log.gz 2026-02-19T21:12:56.153 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121674.log 2026-02-19T21:12:56.154 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67903.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49055.log 2026-02-19T21:12:56.154 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121674.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67903.log.gz 2026-02-19T21:12:56.154 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-19T21:12:56.154 INFO:teuthology.orchestra.run.trial151.stderr: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123983.log 2026-02-19T21:12:56.154 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.121674.log.gz 2026-02-19T21:12:56.154 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49055.log: /var/log/ceph/cluster2-osd.2.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49055.log.gz 2026-02-19T21:12:56.154 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69903.log 2026-02-19T21:12:56.155 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84995.log 2026-02-19T21:12:56.155 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123983.log: /var/log/ceph/cluster1-client.admin.69903.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123983.log.gz 2026-02-19T21:12:56.155 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.admin.69903.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28094.log 2026-02-19T21:12:56.155 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:56.156 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.84995.log: 3.9% -- replaced with /var/log/ceph/cluster2-client.mirror.84995.log.gz 2026-02-19T21:12:56.156 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48923.log 2026-02-19T21:12:56.156 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120284.log 2026-02-19T21:12:56.157 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28094.log: /var/log/ceph/cluster2-client.mirror.48923.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101204.log 2026-02-19T21:12:56.157 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48923.log.gz 2026-02-19T21:12:56.157 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.28094.log.gz 2026-02-19T21:12:56.158 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120284.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.120284.log.gz 2026-02-19T21:12:56.158 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81951.log 2026-02-19T21:12:56.159 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42009.log 2026-02-19T21:12:56.160 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.101204.log: /var/log/ceph/cluster2-client.mirror.81951.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101204.log.gz 2026-02-19T21:12:56.160 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104035.log 2026-02-19T21:12:56.161 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42009.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42009.log.gz 2026-02-19T21:12:56.161 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102118.log 2026-02-19T21:12:56.162 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.104035.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.104035.log.gz 2026-02-19T21:12:56.163 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98993.log 2026-02-19T21:12:56.163 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102118.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102118.log.gz 2026-02-19T21:12:56.164 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31414.log 2026-02-19T21:12:56.164 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98993.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98993.log.gz 2026-02-19T21:12:56.164 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41586.log 2026-02-19T21:12:56.165 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31414.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118890.log 2026-02-19T21:12:56.165 INFO:teuthology.orchestra.run.trial151.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31414.log.gz 2026-02-19T21:12:56.166 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41586.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41586.log.gz 2026-02-19T21:12:56.173 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-19T21:12:56.174 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.102555.log.gz 2026-02-19T21:12:56.174 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118890.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118890.log.gz 2026-02-19T21:12:56.174 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106024.log 2026-02-19T21:12:56.175 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113176.log 2026-02-19T21:12:56.175 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-osd.1.log: /var/log/ceph/cluster2-client.mirror.106024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73328.log 2026-02-19T21:12:56.175 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106024.log.gz 2026-02-19T21:12:56.178 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.113176.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.113176.log.gz 2026-02-19T21:12:56.193 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128263.log 2026-02-19T21:12:56.194 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73328.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73328.log.gz 2026-02-19T21:12:56.194 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53352.log 2026-02-19T21:12:56.196 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.128263.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.128263.log.gz 2026-02-19T21:12:56.196 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.91583.log 2026-02-19T21:12:56.197 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53352.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53352.log.gz 2026-02-19T21:12:56.197 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96611.log 2026-02-19T21:12:56.205 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.1.91583.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72544.log 2026-02-19T21:12:56.206 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96611.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96611.log.gz 2026-02-19T21:12:56.220 INFO:teuthology.orchestra.run.trial151.stderr: 93.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47080.log 2026-02-19T21:12:56.221 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72544.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.72544.log.gz 2026-02-19T21:12:56.221 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.1.91583.log.gz 2026-02-19T21:12:56.222 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106433.log 2026-02-19T21:12:56.222 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47080.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47080.log.gz 2026-02-19T21:12:56.222 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63459.log 2026-02-19T21:12:56.223 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106433.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106433.log.gz 2026-02-19T21:12:56.223 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121422.log 2026-02-19T21:12:56.224 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63459.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63459.log.gz 2026-02-19T21:12:56.224 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31705.log 2026-02-19T21:12:56.226 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121422.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121422.log.gz 2026-02-19T21:12:56.226 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50770.log 2026-02-19T21:12:56.227 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31705.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68326.log 2026-02-19T21:12:56.227 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31705.log.gz 2026-02-19T21:12:56.228 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50770.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50770.log.gz 91.1% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-19T21:12:56.228 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:56.229 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59319.log 2026-02-19T21:12:56.229 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.68326.log: /var/log/ceph/cluster2-client.mirror.109813.log 2026-02-19T21:12:56.229 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68326.log.gz 2026-02-19T21:12:56.230 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59319.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77270.log 2026-02-19T21:12:56.230 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109813.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59319.log.gz 2026-02-19T21:12:56.230 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.109813.log.gz 2026-02-19T21:12:56.231 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51990.log 2026-02-19T21:12:56.232 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121130.log 2026-02-19T21:12:56.233 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77270.log: /var/log/ceph/cluster2-client.mirror.51990.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51990.log.gz 2026-02-19T21:12:56.233 INFO:teuthology.orchestra.run.trial151.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77270.log.gz 2026-02-19T21:12:56.233 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44727.log 2026-02-19T21:12:56.233 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106614.log 2026-02-19T21:12:56.233 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121130.log.gz 2026-02-19T21:12:56.233 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44727.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88649.log 2026-02-19T21:12:56.235 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106614.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44727.log.gz 2026-02-19T21:12:56.235 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106614.log.gz 2026-02-19T21:12:56.235 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60972.log 2026-02-19T21:12:56.235 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114736.log 2026-02-19T21:12:56.235 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88649.log: /var/log/ceph/cluster2-client.mirror.60972.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88649.log.gz 89.0% 2026-02-19T21:12:56.235 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.60972.log.gz 2026-02-19T21:12:56.235 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64561.log 2026-02-19T21:12:56.237 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114736.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52611.log 2026-02-19T21:12:56.237 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114736.log.gz 2026-02-19T21:12:56.237 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64561.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118389.log 2026-02-19T21:12:56.237 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52611.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64561.log.gz 2026-02-19T21:12:56.237 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52611.log.gz 2026-02-19T21:12:56.237 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112229.log 2026-02-19T21:12:56.238 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76749.log 2026-02-19T21:12:56.238 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118389.log: /var/log/ceph/cluster2-client.mirror.112229.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112229.log.gz 2026-02-19T21:12:56.238 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114346.log 2026-02-19T21:12:56.241 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76749.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76749.log.gz 2026-02-19T21:12:56.241 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48667.log 2026-02-19T21:12:56.242 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114346.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114346.log.gz 2026-02-19T21:12:56.242 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90972.log 2026-02-19T21:12:56.243 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48667.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48667.log.gz 2026-02-19T21:12:56.243 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46310.log 2026-02-19T21:12:56.243 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90972.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.90972.log.gz 2026-02-19T21:12:56.244 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21306.log 2026-02-19T21:12:56.244 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46310.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46310.log.gz 2026-02-19T21:12:56.245 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87394.log 2026-02-19T21:12:56.245 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.21306.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.21306.log.gz 2026-02-19T21:12:56.246 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62780.log 2026-02-19T21:12:56.246 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87394.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87394.log.gz 2026-02-19T21:12:56.246 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27353.log 2026-02-19T21:12:56.247 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62780.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62780.log.gz 2026-02-19T21:12:56.247 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99276.log 2026-02-19T21:12:56.248 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27353.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27353.log.gz 2026-02-19T21:12:56.248 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80473.log 2026-02-19T21:12:56.248 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99276.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99276.log.gz 2026-02-19T21:12:56.249 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111873.log 2026-02-19T21:12:56.250 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.80473.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80473.log.gz 2026-02-19T21:12:56.250 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74880.log 2026-02-19T21:12:56.251 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111873.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111873.log.gz 2026-02-19T21:12:56.251 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127100.log 2026-02-19T21:12:56.252 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74880.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74880.log.gz 2026-02-19T21:12:56.252 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51261.log 2026-02-19T21:12:56.253 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127100.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127100.log.gz 2026-02-19T21:12:56.254 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122045.log 2026-02-19T21:12:56.262 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51261.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92760.log.gz 2026-02-19T21:12:56.265 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67612.log 2026-02-19T21:12:56.266 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122045.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122045.log.gz 2026-02-19T21:12:56.267 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30641.log 2026-02-19T21:12:56.267 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67612.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67612.log.gz 2026-02-19T21:12:56.267 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33251.log 2026-02-19T21:12:56.268 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.30641.log: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.30641.log.gz 2026-02-19T21:12:56.268 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118861.log 2026-02-19T21:12:56.269 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33251.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33251.log.gz 2026-02-19T21:12:56.269 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25475.log 2026-02-19T21:12:56.271 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118861.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.118861.log.gz 2026-02-19T21:12:56.271 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82729.log 2026-02-19T21:12:56.272 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34706.log 2026-02-19T21:12:56.272 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25475.log.gz 2026-02-19T21:12:56.286 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82729.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.86364.log 2026-02-19T21:12:56.286 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34706.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34706.log.gz 2026-02-19T21:12:56.288 INFO:teuthology.orchestra.run.trial151.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82729.log.gz 2026-02-19T21:12:56.289 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37688.log 2026-02-19T21:12:56.294 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.86364.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.95976.log.gz 2026-02-19T21:12:56.295 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125584.log 2026-02-19T21:12:56.296 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.37688.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.37688.log.gz 2026-02-19T21:12:56.302 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.86364.log.gz 2026-02-19T21:12:56.302 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39937.log 2026-02-19T21:12:56.303 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122065.log 2026-02-19T21:12:56.304 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125584.log: /var/log/ceph/cluster2-client.mirror.39937.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125584.log.gz 2026-02-19T21:12:56.304 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39937.log.gz 2026-02-19T21:12:56.304 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.79939.log 2026-02-19T21:12:56.304 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122065.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86037.log 2026-02-19T21:12:56.305 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.3.79939.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.122065.log.gz 2026-02-19T21:12:56.305 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123758.log 2026-02-19T21:12:56.318 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86037.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102763.log 2026-02-19T21:12:56.319 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123758.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123758.log.gz 2026-02-19T21:12:56.334 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30726.log 2026-02-19T21:12:56.335 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102763.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102763.log.gz 2026-02-19T21:12:56.341 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75136.log 2026-02-19T21:12:56.349 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30726.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82171.log.gz 2026-02-19T21:12:56.349 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92208.log 2026-02-19T21:12:56.351 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75136.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75136.log.gz 2026-02-19T21:12:56.351 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.30726.log.gz 2026-02-19T21:12:56.352 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51637.log 2026-02-19T21:12:56.352 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92208.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92208.log.gz 2026-02-19T21:12:56.352 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123376.log 2026-02-19T21:12:56.353 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51637.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51637.log.gz 2026-02-19T21:12:56.353 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26577.log 2026-02-19T21:12:56.354 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123376.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123376.log.gz 2026-02-19T21:12:56.354 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82148.log 2026-02-19T21:12:56.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26577.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26577.log.gz 2026-02-19T21:12:56.356 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22193.log 2026-02-19T21:12:56.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82148.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82148.log.gz 2026-02-19T21:12:56.356 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75625.log 2026-02-19T21:12:56.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22193.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22193.log.gz 2026-02-19T21:12:56.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64949.log 2026-02-19T21:12:56.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.75625.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75625.log.gz 2026-02-19T21:12:56.360 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73134.log 2026-02-19T21:12:56.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64949.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64949.log.gz 2026-02-19T21:12:56.361 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73910.log 2026-02-19T21:12:56.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73134.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73134.log.gz 2026-02-19T21:12:56.361 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35905.log 2026-02-19T21:12:56.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73910.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73910.log.gz 2026-02-19T21:12:56.363 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112445.log 2026-02-19T21:12:56.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35905.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35905.log.gz 2026-02-19T21:12:56.364 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123308.log 2026-02-19T21:12:56.365 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.112445.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.112445.log.gz 2026-02-19T21:12:56.365 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124433.log 2026-02-19T21:12:56.366 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123308.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123308.log.gz 2026-02-19T21:12:56.366 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90091.log 2026-02-19T21:12:56.366 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124433.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124433.log.gz 2026-02-19T21:12:56.367 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79275.log 2026-02-19T21:12:56.367 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90091.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90091.log.gz 2026-02-19T21:12:56.367 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109024.log 2026-02-19T21:12:56.368 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79275.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79275.log.gz 2026-02-19T21:12:56.368 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65091.log 2026-02-19T21:12:56.369 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.109024.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.109024.log.gz 2026-02-19T21:12:56.370 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19176.log 2026-02-19T21:12:56.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65091.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86037.log.gz 2026-02-19T21:12:56.371 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75379.log 2026-02-19T21:12:56.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.19176.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19176.log.gz 2026-02-19T21:12:56.372 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69584.log 2026-02-19T21:12:56.372 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75379.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75379.log.gz 2026-02-19T21:12:56.372 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90554.log 2026-02-19T21:12:56.374 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69584.log: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65091.log.gz 2026-02-19T21:12:56.376 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.69584.log.gz 2026-02-19T21:12:56.377 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58120.log 2026-02-19T21:12:56.377 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90554.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93735.log 2026-02-19T21:12:56.377 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58120.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90554.log.gz 2026-02-19T21:12:56.377 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58120.log.gz 2026-02-19T21:12:56.377 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19890.log 2026-02-19T21:12:56.378 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93735.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106182.log 2026-02-19T21:12:56.379 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19890.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93735.log.gz 2026-02-19T21:12:56.379 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58155.log 2026-02-19T21:12:56.379 INFO:teuthology.orchestra.run.trial151.stderr: 68.4% -- replaced with /var/log/ceph/cluster1-client.mirror.19890.log.gz 2026-02-19T21:12:56.379 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.106182.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76385.log 2026-02-19T21:12:56.380 INFO:teuthology.orchestra.run.trial151.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.58155.log: -- replaced with /var/log/ceph/cluster1-client.mirror.106182.log.gz 2026-02-19T21:12:56.380 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58155.log.gz 2026-02-19T21:12:56.380 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30038.log 2026-02-19T21:12:56.382 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76385.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76385.log.gz 2026-02-19T21:12:56.383 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119833.log 2026-02-19T21:12:56.383 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30038.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116848.log 2026-02-19T21:12:56.384 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119833.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30038.log.gz 2026-02-19T21:12:56.384 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.119833.log.gz 2026-02-19T21:12:56.384 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49152.log 2026-02-19T21:12:56.384 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116848.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126417.log 2026-02-19T21:12:56.384 INFO:teuthology.orchestra.run.trial151.stderr: 88.4%/var/log/ceph/cluster2-client.mirror.49152.log: -- replaced with /var/log/ceph/cluster2-client.mirror.116848.log.gz 2026-02-19T21:12:56.385 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49152.log.gz 2026-02-19T21:12:56.385 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101184.log 2026-02-19T21:12:56.385 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126417.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110687.log 2026-02-19T21:12:56.385 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101184.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126417.log.gz 2026-02-19T21:12:56.386 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113601.log 2026-02-19T21:12:56.386 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110687.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110687.log.gz 2026-02-19T21:12:56.386 INFO:teuthology.orchestra.run.trial151.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.101184.log.gz 2026-02-19T21:12:56.386 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98128.log 2026-02-19T21:12:56.387 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113601.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108285.log 2026-02-19T21:12:56.387 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98128.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113601.log.gz 2026-02-19T21:12:56.388 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98128.log.gz 2026-02-19T21:12:56.388 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31352.log 2026-02-19T21:12:56.388 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108285.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.108285.log.gz 2026-02-19T21:12:56.388 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73037.log 2026-02-19T21:12:56.389 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31352.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63944.log 2026-02-19T21:12:56.389 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73037.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91088.log 2026-02-19T21:12:56.389 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31352.log.gz 2026-02-19T21:12:56.389 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.73037.log.gz 2026-02-19T21:12:56.389 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63944.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63944.log.gz 2026-02-19T21:12:56.390 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74104.log 2026-02-19T21:12:56.390 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.91088.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100316.log 2026-02-19T21:12:56.390 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 has been restored 2026-02-19T21:12:56.390 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.mon.a has been restored 2026-02-19T21:12:56.391 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74104.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.91088.log.gz 2026-02-19T21:12:56.391 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74104.log.gz 2026-02-19T21:12:56.391 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42300.log 2026-02-19T21:12:56.391 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100316.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100316.log.gz 2026-02-19T21:12:56.392 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106666.log 2026-02-19T21:12:56.392 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42300.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87528.log 2026-02-19T21:12:56.392 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106666.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42300.log.gz 2026-02-19T21:12:56.392 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106666.log.gz 2026-02-19T21:12:56.392 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20180.log 2026-02-19T21:12:56.393 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87528.log: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.84340.log.gz 2026-02-19T21:12:56.393 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35482.log 2026-02-19T21:12:56.393 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36487.log 2026-02-19T21:12:56.394 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35482.log: 55.6% -- replaced with /var/log/ceph/cluster2-client.mirror.20180.log.gz 2026-02-19T21:12:56.394 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24152.log 2026-02-19T21:12:56.394 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.35482.log.gz 2026-02-19T21:12:56.395 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36487.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36487.log.gz 2026-02-19T21:12:56.395 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86841.log 2026-02-19T21:12:56.395 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98448.log 2026-02-19T21:12:56.395 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.86841.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24152.log.gz 2026-02-19T21:12:56.396 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118174.log 2026-02-19T21:12:56.396 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98448.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98448.log.gz 2026-02-19T21:12:56.397 INFO:teuthology.orchestra.run.trial151.stderr: 92.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73487.log 2026-02-19T21:12:56.397 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.86841.log.gz 2026-02-19T21:12:56.397 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118174.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118174.log.gz 2026-02-19T21:12:56.397 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125032.log 2026-02-19T21:12:56.398 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73487.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102456.log 2026-02-19T21:12:56.398 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73487.log.gz 2026-02-19T21:12:56.398 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125032.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35579.log 2026-02-19T21:12:56.399 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102456.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102456.log.gz 2026-02-19T21:12:56.399 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64852.log 2026-02-19T21:12:56.400 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35579.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35579.log.gz 2026-02-19T21:12:56.400 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121402.log 2026-02-19T21:12:56.400 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64852.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64852.log.gz 2026-02-19T21:12:56.401 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57150.log 2026-02-19T21:12:56.401 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121402.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121402.log.gz 2026-02-19T21:12:56.402 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71570.log 2026-02-19T21:12:56.402 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57150.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57150.log.gz 2026-02-19T21:12:56.403 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66157.log 2026-02-19T21:12:56.403 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71570.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71570.log.gz 2026-02-19T21:12:56.404 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22658.log 2026-02-19T21:12:56.404 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66157.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66157.log.gz 2026-02-19T21:12:56.404 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69164.log 2026-02-19T21:12:56.405 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22658.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22658.log.gz 2026-02-19T21:12:56.405 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66351.log 2026-02-19T21:12:56.406 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69164.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69164.log.gz 2026-02-19T21:12:56.406 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118630.log 2026-02-19T21:12:56.407 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66351.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.66351.log.gz 2026-02-19T21:12:56.407 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65963.log 2026-02-19T21:12:56.408 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118630.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.118630.log.gz 2026-02-19T21:12:56.409 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74457.log 2026-02-19T21:12:56.409 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65963.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65963.log.gz 2026-02-19T21:12:56.410 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22809.log 2026-02-19T21:12:56.410 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74457.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74457.log.gz 2026-02-19T21:12:56.410 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42071.log 2026-02-19T21:12:56.411 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22809.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22809.log.gz 2026-02-19T21:12:56.411 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116630.log 2026-02-19T21:12:56.412 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42071.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42071.log.gz 2026-02-19T21:12:56.412 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119125.log 2026-02-19T21:12:56.413 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.116630.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.116630.log.gz 2026-02-19T21:12:56.414 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94641.log 2026-02-19T21:12:56.414 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119125.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.119125.log.gz 2026-02-19T21:12:56.415 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70344.log 2026-02-19T21:12:56.415 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94641.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94641.log.gz 2026-02-19T21:12:56.415 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43237.log 2026-02-19T21:12:56.416 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70344.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70344.log.gz 2026-02-19T21:12:56.416 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77986.log 2026-02-19T21:12:56.417 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43237.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43237.log.gz 2026-02-19T21:12:56.417 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93937.log 2026-02-19T21:12:56.419 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.77986.log: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.77986.log.gz 2026-02-19T21:12:56.419 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100408.log 2026-02-19T21:12:56.419 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93937.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93937.log.gz 2026-02-19T21:12:56.420 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67841.log 2026-02-19T21:12:56.421 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100408.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.100408.log.gz 2026-02-19T21:12:56.422 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.111741.log.gz 2026-02-19T21:12:56.422 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111453.log 2026-02-19T21:12:56.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67841.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77950.log.gz 2026-02-19T21:12:56.422 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67841.log.gz 2026-02-19T21:12:56.422 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62295.log 2026-02-19T21:12:56.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111453.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19251.log 2026-02-19T21:12:56.423 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.111453.log.gz 2026-02-19T21:12:56.423 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43369.log 2026-02-19T21:12:56.423 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62295.log.gz/var/log/ceph/cluster1-client.admin.19251.log: 2026-02-19T21:12:56.423 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84080.log 2026-02-19T21:12:56.423 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19251.log.gz 2026-02-19T21:12:56.424 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78186.log 2026-02-19T21:12:56.424 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.84080.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43369.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25572.log 2026-02-19T21:12:56.424 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:56.424 INFO:teuthology.orchestra.run.trial151.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84080.log.gz 2026-02-19T21:12:56.424 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61034.log 2026-02-19T21:12:56.424 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60060.log 2026-02-19T21:12:56.425 INFO:teuthology.orchestra.run.trial151.stderr: 88.1%/var/log/ceph/cluster2-client.mirror.61034.log: -- replaced with /var/log/ceph/cluster2-client.mirror.25572.log.gz 2026-02-19T21:12:56.425 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61034.log.gz 2026-02-19T21:12:56.425 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48341.log 2026-02-19T21:12:56.426 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60060.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60060.log.gz 2026-02-19T21:12:56.426 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123578.log 2026-02-19T21:12:56.426 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48341.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33542.log 2026-02-19T21:12:56.427 INFO:teuthology.orchestra.run.trial151.stderr: 88.8%/var/log/ceph/cluster1-client.mirror.123578.log: -- replaced with /var/log/ceph/cluster2-client.mirror.48341.log.gz 2026-02-19T21:12:56.427 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123578.log.gz 2026-02-19T21:12:56.427 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67744.log 2026-02-19T21:12:56.427 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33542.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24214.log 2026-02-19T21:12:56.428 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67744.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33542.log.gz 2026-02-19T21:12:56.428 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67744.log.gz 2026-02-19T21:12:56.428 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71896.log 2026-02-19T21:12:56.428 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24214.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33577.log 2026-02-19T21:12:56.429 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71896.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24214.log.gz 2026-02-19T21:12:56.429 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71896.log.gz 2026-02-19T21:12:56.429 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50188.log 2026-02-19T21:12:56.429 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33577.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36778.log 2026-02-19T21:12:56.430 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50188.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33577.log.gz 2026-02-19T21:12:56.430 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50188.log.gz 2026-02-19T21:12:56.430 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42459.log 2026-02-19T21:12:56.430 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36778.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79666.log 2026-02-19T21:12:56.431 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42459.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36778.log.gz 2026-02-19T21:12:56.431 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127256.log 2026-02-19T21:12:56.431 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42459.log.gz 2026-02-19T21:12:56.431 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79666.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79666.log.gz 2026-02-19T21:12:56.433 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78978.log 2026-02-19T21:12:56.433 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127256.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.51288.log 2026-02-19T21:12:56.434 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78978.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.127256.log.gz 2026-02-19T21:12:56.443 INFO:teuthology.orchestra.run.trial151.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78978.log.gz 2026-02-19T21:12:56.444 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83793.log 2026-02-19T21:12:56.444 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.51288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106758.log 2026-02-19T21:12:56.446 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83793.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.0.51288.log.gz 2026-02-19T21:12:56.454 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78236.log.gz 2026-02-19T21:12:56.457 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44824.log 2026-02-19T21:12:56.457 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61748.log 2026-02-19T21:12:56.458 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106758.log: /var/log/ceph/cluster2-client.mirror.44824.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44824.log.gz 2026-02-19T21:12:56.459 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52999.log 2026-02-19T21:12:56.459 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61748.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.106758.log.gz 2026-02-19T21:12:56.459 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61748.log.gz 2026-02-19T21:12:56.459 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70697.log 2026-02-19T21:12:56.460 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56568.log 2026-02-19T21:12:56.460 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52999.log: /var/log/ceph/cluster2-client.mirror.70697.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52999.log.gz 2026-02-19T21:12:56.460 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70697.log.gz 2026-02-19T21:12:56.460 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89788.log 2026-02-19T21:12:56.461 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68811.log 2026-02-19T21:12:56.461 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56568.log: /var/log/ceph/cluster2-client.mirror.89788.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56568.log.gz 2026-02-19T21:12:56.461 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89788.log.gz 2026-02-19T21:12:56.462 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42981.log 2026-02-19T21:12:56.462 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74201.log 2026-02-19T21:12:56.462 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78289.log.gz 2026-02-19T21:12:56.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68811.log: /var/log/ceph/cluster2-client.mirror.42981.log: 89.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107826.log 2026-02-19T21:12:56.463 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.68811.log.gz 2026-02-19T21:12:56.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74201.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42981.log.gz 2026-02-19T21:12:56.463 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74201.log.gz 2026-02-19T21:12:56.463 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59769.log 2026-02-19T21:12:56.464 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107826.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103391.log 2026-02-19T21:12:56.464 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59769.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107826.log.gz 2026-02-19T21:12:56.464 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51408.log 2026-02-19T21:12:56.464 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59769.log.gz 2026-02-19T21:12:56.464 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40554.log 2026-02-19T21:12:56.465 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103391.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.103391.log.gz 2026-02-19T21:12:56.465 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124276.log 2026-02-19T21:12:56.465 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51408.log: /var/log/ceph/cluster2-client.mirror.40554.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108165.log 2026-02-19T21:12:56.465 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51408.log.gz 2026-02-19T21:12:56.466 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124276.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40554.log.gz 2026-02-19T21:12:56.466 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124276.log.gz 2026-02-19T21:12:56.466 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128867.log 2026-02-19T21:12:56.466 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77674.log 2026-02-19T21:12:56.466 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108165.log.gz 2026-02-19T21:12:56.467 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128867.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43722.log 2026-02-19T21:12:56.467 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128867.log.gz 2026-02-19T21:12:56.467 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90183.log 2026-02-19T21:12:56.467 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77674.log: /var/log/ceph/cluster2-client.mirror.43722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16130.log 2026-02-19T21:12:56.467 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77674.log.gz 2026-02-19T21:12:56.467 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90183.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43722.log.gz 2026-02-19T21:12:56.468 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29782.log 2026-02-19T21:12:56.468 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.16130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32805.log 2026-02-19T21:12:56.468 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16130.log.gz 2026-02-19T21:12:56.468 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29782.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90183.log.gz 2026-02-19T21:12:56.469 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29782.log.gz 2026-02-19T21:12:56.469 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125923.log 2026-02-19T21:12:56.469 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41877.log 2026-02-19T21:12:56.470 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32805.log: /var/log/ceph/cluster1-client.mirror.125923.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108079.log 2026-02-19T21:12:56.470 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% 77.4% -- replaced with /var/log/ceph/cluster2-client.mirror.32805.log.gz 2026-02-19T21:12:56.470 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.125923.log.gz 2026-02-19T21:12:56.470 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41877.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123916.log 2026-02-19T21:12:56.471 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38034.log 2026-02-19T21:12:56.471 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41877.log.gz 2026-02-19T21:12:56.471 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108079.log: /var/log/ceph/cluster1-client.mirror.123916.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108079.log.gz 2026-02-19T21:12:56.471 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123916.log.gz 2026-02-19T21:12:56.471 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51015.log 2026-02-19T21:12:56.471 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114684.log 2026-02-19T21:12:56.472 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38034.log: /var/log/ceph/cluster1-client.mirror.51015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62683.log 2026-02-19T21:12:56.472 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.51015.log.gz 2026-02-19T21:12:56.472 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114684.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38034.log.gz 2026-02-19T21:12:56.485 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118699.log 2026-02-19T21:12:56.486 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62683.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120597.log 2026-02-19T21:12:56.486 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62683.log.gz 2026-02-19T21:12:56.486 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118699.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118699.log.gz 2026-02-19T21:12:56.486 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26930.log 2026-02-19T21:12:56.487 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53962.log 2026-02-19T21:12:56.487 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120597.log: /var/log/ceph/cluster2-client.mirror.26930.log: 89.2% 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.26930.log.gz 2026-02-19T21:12:56.487 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.120597.log.gz 2026-02-19T21:12:56.488 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31511.log 2026-02-19T21:12:56.489 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53962.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.79226.log.gz 2026-02-19T21:12:56.489 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31123.log 2026-02-19T21:12:56.490 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31511.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31511.log.gz 2026-02-19T21:12:56.490 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32225.log 2026-02-19T21:12:56.490 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31123.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83071.log 2026-02-19T21:12:56.491 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32225.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62101.log 2026-02-19T21:12:56.491 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83071.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32225.log.gz 2026-02-19T21:12:56.491 INFO:teuthology.orchestra.run.trial151.stderr: 84.8%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.83071.log.gz -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41718.log 2026-02-19T21:12:56.492 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:56.492 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53962.log.gz 2026-02-19T21:12:56.492 INFO:teuthology.orchestra.run.trial151.stderr: 88.5%/var/log/ceph/cluster2-client.mirror.62101.log: -- replaced with /var/log/ceph/cluster2-client.mirror.31123.log.gz 2026-02-19T21:12:56.492 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57282.log 2026-02-19T21:12:56.492 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62101.log.gz 2026-02-19T21:12:56.492 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41718.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118441.log 2026-02-19T21:12:56.492 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57282.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81349.log 2026-02-19T21:12:56.492 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41718.log.gz 2026-02-19T21:12:56.493 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118441.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57282.log.gz 2026-02-19T21:12:56.493 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118441.log.gz 2026-02-19T21:12:56.493 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104657.log 2026-02-19T21:12:56.493 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121873.log 2026-02-19T21:12:56.493 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81349.log: /var/log/ceph/cluster1-client.mirror.104657.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72733.log 2026-02-19T21:12:56.494 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121873.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121873.log.gz 2026-02-19T21:12:56.495 INFO:teuthology.orchestra.run.trial151.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.104657.log.gz 2026-02-19T21:12:56.495 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63591.log 2026-02-19T21:12:56.495 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72733.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55730.log 2026-02-19T21:12:56.496 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63591.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63591.log.gz 2026-02-19T21:12:56.496 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100861.log 2026-02-19T21:12:56.497 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55730.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55730.log.gz 2026-02-19T21:12:56.497 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38381.log 2026-02-19T21:12:56.497 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100861.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100861.log.gz 2026-02-19T21:12:56.498 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37034.log 2026-02-19T21:12:56.498 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38381.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38381.log.gz 2026-02-19T21:12:56.499 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92451.log 2026-02-19T21:12:56.499 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72733.log.gz 2026-02-19T21:12:56.499 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37034.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56083.log 2026-02-19T21:12:56.499 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92451.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37034.log.gz 2026-02-19T21:12:56.500 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92451.log.gz 2026-02-19T21:12:56.500 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49218.log 2026-02-19T21:12:56.500 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56083.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95667.log 2026-02-19T21:12:56.500 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49218.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56083.log.gz 2026-02-19T21:12:56.501 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49218.log.gz 2026-02-19T21:12:56.501 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118080.log 2026-02-19T21:12:56.501 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110563.log 2026-02-19T21:12:56.501 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95667.log.gz 2026-02-19T21:12:56.502 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118080.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82662.log 2026-02-19T21:12:56.502 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110563.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118080.log.gz 2026-02-19T21:12:56.502 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110563.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86863.log 2026-02-19T21:12:56.502 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:56.502 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82662.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120039.log 2026-02-19T21:12:56.533 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86863.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32031.log 2026-02-19T21:12:56.534 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120039.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.120039.log.gz 2026-02-19T21:12:56.558 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124816.log 2026-02-19T21:12:56.558 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32031.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32031.log.gz 2026-02-19T21:12:56.565 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117018.log 2026-02-19T21:12:56.566 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124816.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124816.log.gz 2026-02-19T21:12:56.578 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.104679.log.gz 2026-02-19T21:12:56.578 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111719.log 2026-02-19T21:12:56.579 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117018.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117018.log.gz 2026-02-19T21:12:56.580 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61907.log 2026-02-19T21:12:56.581 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.111719.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.111719.log.gz 2026-02-19T21:12:56.581 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92580.log 2026-02-19T21:12:56.582 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61907.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61907.log.gz 2026-02-19T21:12:56.582 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92855.log 2026-02-19T21:12:56.583 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92580.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.92580.log.gz 2026-02-19T21:12:56.583 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127928.log 2026-02-19T21:12:56.583 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92855.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92855.log.gz 2026-02-19T21:12:56.584 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65415.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50382.log 2026-02-19T21:12:56.584 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:56.585 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127928.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.127928.log.gz 2026-02-19T21:12:56.586 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93983.log 2026-02-19T21:12:56.586 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50382.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31035.log 2026-02-19T21:12:56.586 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93983.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50382.log.gz 2026-02-19T21:12:56.586 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93983.log.gz 2026-02-19T21:12:56.586 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37131.log 2026-02-19T21:12:56.590 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.31035.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.31035.log.gz 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64270.log 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37131.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110652.log 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64270.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37131.log.gz 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127304.log.gz 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64270.log.gz 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104748.log 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110652.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102852.log 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.104748.log: -- replaced with /var/log/ceph/cluster2-client.mirror.110652.log.gz 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88945.log 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104748.log.gz 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102852.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77568.log 2026-02-19T21:12:56.591 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102852.log.gz 2026-02-19T21:12:56.592 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88945.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46789.log 2026-02-19T21:12:56.592 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77568.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100141.log 2026-02-19T21:12:56.592 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46789.log: 88.8% 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77568.log.gz 2026-02-19T21:12:56.592 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46789.log.gz 2026-02-19T21:12:56.593 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117068.log 2026-02-19T21:12:56.595 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100141.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55792.log 2026-02-19T21:12:56.595 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117068.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100141.log.gz 2026-02-19T21:12:56.595 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117068.log.gz 2026-02-19T21:12:56.595 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119103.log 2026-02-19T21:12:56.595 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55792.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125754.log 2026-02-19T21:12:56.595 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119103.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55792.log.gz 2026-02-19T21:12:56.599 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46019.log 2026-02-19T21:12:56.599 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125754.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.119103.log.gz 2026-02-19T21:12:56.599 INFO:teuthology.orchestra.run.trial151.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125754.log.gz 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47794.log 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59478.log 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47794.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46019.log.gz 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47794.log.gz 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117153.log 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59478.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74360.log 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117153.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59478.log.gz 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117153.log.gz 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27256.log 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74360.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120852.log 2026-02-19T21:12:56.600 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27256.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74360.log.gz 2026-02-19T21:12:56.601 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27256.log.gz 2026-02-19T21:12:56.605 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29200.log 2026-02-19T21:12:56.605 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120852.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92254.log 2026-02-19T21:12:56.605 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120852.log.gz/var/log/ceph/cluster2-client.mirror.29200.log: 2026-02-19T21:12:56.605 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29200.log.gz 2026-02-19T21:12:56.605 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71993.log 2026-02-19T21:12:56.605 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118770.log 2026-02-19T21:12:56.605 INFO:teuthology.orchestra.run.trial151.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.71993.log: -- replaced with /var/log/ceph/cluster2-client.mirror.92254.log.gz 2026-02-19T21:12:56.605 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71993.log.gz 2026-02-19T21:12:56.606 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55501.log 2026-02-19T21:12:56.606 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118770.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56215.log 2026-02-19T21:12:56.606 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55501.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118770.log.gz 2026-02-19T21:12:56.606 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55501.log.gz 2026-02-19T21:12:56.606 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88903.log 2026-02-19T21:12:56.606 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72708.log 2026-02-19T21:12:56.606 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88903.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56215.log.gz 2026-02-19T21:12:56.606 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51893.log 2026-02-19T21:12:56.606 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.72708.log: 88.8% 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72708.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.88903.log.gz 2026-02-19T21:12:56.606 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66254.log 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51893.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89129.log 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66254.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51893.log.gz 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66254.log.gz 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124793.log 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89129.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64173.log 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124793.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.89129.log.gz 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124793.log.gz 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119754.log 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64173.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77375.log 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119754.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64173.log.gz 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21786.log 2026-02-19T21:12:56.610 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77375.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77375.log.gz 2026-02-19T21:12:56.611 INFO:teuthology.orchestra.run.trial151.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119754.log.gz 2026-02-19T21:12:56.611 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119691.log 2026-02-19T21:12:56.612 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21786.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21786.log.gz 2026-02-19T21:12:56.612 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47018.log 2026-02-19T21:12:56.612 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119691.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119691.log.gz 2026-02-19T21:12:56.612 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93700.log 2026-02-19T21:12:56.612 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47018.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47018.log.gz 2026-02-19T21:12:56.613 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24699.log 2026-02-19T21:12:56.613 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93700.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93700.log.gz 2026-02-19T21:12:56.614 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87734.log 2026-02-19T21:12:56.614 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24699.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24699.log.gz 2026-02-19T21:12:56.614 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-19T21:12:56.618 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.87734.log: 95.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87734.log.gz 2026-02-19T21:12:56.618 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56409.log 2026-02-19T21:12:56.618 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91583.log 2026-02-19T21:12:56.618 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56409.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56409.log.gz 2026-02-19T21:12:56.619 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38769.log 2026-02-19T21:12:56.621 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.91583.log: 92.4% 87.1% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-19T21:12:56.622 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.91583.log.gz 2026-02-19T21:12:56.622 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99638.log 2026-02-19T21:12:56.622 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38769.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112979.log 2026-02-19T21:12:56.623 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99638.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38769.log.gz 2026-02-19T21:12:56.624 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99638.log.gz 2026-02-19T21:12:56.624 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38610.log 2026-02-19T21:12:56.624 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38707.log 2026-02-19T21:12:56.624 INFO:teuthology.orchestra.run.trial151.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.38610.log: -- replaced with /var/log/ceph/cluster2-client.mirror.112979.log.gz 2026-02-19T21:12:56.625 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38610.log.gz 2026-02-19T21:12:56.625 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96949.log 2026-02-19T21:12:56.625 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69627.log 2026-02-19T21:12:56.625 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96949.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38707.log.gz 2026-02-19T21:12:56.626 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79298.log 2026-02-19T21:12:56.626 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96949.log.gz 2026-02-19T21:12:56.626 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69627.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41489.log 2026-02-19T21:12:56.630 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79298.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.69627.log.gz 2026-02-19T21:12:56.633 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124501.log 2026-02-19T21:12:56.635 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41489.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70988.log 2026-02-19T21:12:56.635 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41489.log.gz 2026-02-19T21:12:56.636 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124501.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124501.log.gz 2026-02-19T21:12:56.636 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101159.log 2026-02-19T21:12:56.636 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70988.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.70988.log.gz 2026-02-19T21:12:56.637 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101270.log 2026-02-19T21:12:56.638 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.101159.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.101159.log.gz 2026-02-19T21:12:56.638 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108200.log 2026-02-19T21:12:56.638 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101270.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101270.log.gz 2026-02-19T21:12:56.638 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126101.log 2026-02-19T21:12:56.640 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108200.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108200.log.gz 2026-02-19T21:12:56.640 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87429.log 2026-02-19T21:12:56.640 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126101.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.126101.log.gz 2026-02-19T21:12:56.640 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103302.log 2026-02-19T21:12:56.642 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87429.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87429.log.gz 2026-02-19T21:12:56.642 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75298.log 2026-02-19T21:12:56.642 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103302.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103302.log.gz 2026-02-19T21:12:56.642 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28777.log 2026-02-19T21:12:56.643 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75298.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75298.log.gz 2026-02-19T21:12:56.643 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.84796.log 2026-02-19T21:12:56.644 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28777.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28777.log.gz 2026-02-19T21:12:56.644 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118045.log 2026-02-19T21:12:56.657 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.1.84796.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.1.84796.log.gz 2026-02-19T21:12:56.657 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.79689.log.gz 2026-02-19T21:12:56.657 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40810.log 2026-02-19T21:12:56.658 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122678.log 2026-02-19T21:12:56.658 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118045.log: /var/log/ceph/cluster2-client.mirror.40810.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118045.log.gz 2026-02-19T21:12:56.658 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40810.log.gz 2026-02-19T21:12:56.659 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94446.log 2026-02-19T21:12:56.660 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122678.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19788.log 2026-02-19T21:12:56.660 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.122678.log.gz 2026-02-19T21:12:56.660 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94446.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118538.log 2026-02-19T21:12:56.660 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19788.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94446.log.gz 2026-02-19T21:12:56.661 INFO:teuthology.orchestra.run.trial151.stderr: 63.5% -- replaced with /var/log/ceph/cluster1-client.mirror.19788.log.gz 2026-02-19T21:12:56.661 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109845.log 2026-02-19T21:12:56.661 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118538.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118538.log.gz 2026-02-19T21:12:56.661 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93654.log 2026-02-19T21:12:56.662 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.109845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40034.log 2026-02-19T21:12:56.662 INFO:teuthology.orchestra.run.trial151.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.93654.log: -- replaced with /var/log/ceph/cluster1-client.mirror.109845.log.gz 2026-02-19T21:12:56.662 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93654.log.gz 2026-02-19T21:12:56.662 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94676.log 2026-02-19T21:12:56.662 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40034.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19839.log 2026-02-19T21:12:56.663 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94676.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40034.log.gz 2026-02-19T21:12:56.663 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.94676.log.gz 2026-02-19T21:12:56.663 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50417.log 2026-02-19T21:12:56.663 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19839.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19029.log 2026-02-19T21:12:56.664 INFO:teuthology.orchestra.run.trial151.stderr: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.19839.log.gz 2026-02-19T21:12:56.664 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50417.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22100.log 2026-02-19T21:12:56.664 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19029.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50417.log.gz 2026-02-19T21:12:56.664 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19029.log.gz 2026-02-19T21:12:56.664 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121581.log 2026-02-19T21:12:56.665 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43819.log 2026-02-19T21:12:56.665 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121581.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22100.log.gz 2026-02-19T21:12:56.665 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.83360.log 2026-02-19T21:12:56.665 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43819.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121581.log.gz 2026-02-19T21:12:56.666 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43819.log.gz 2026-02-19T21:12:56.681 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121280.log 2026-02-19T21:12:56.682 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.3.83360.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126736.log 2026-02-19T21:12:56.683 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121280.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121280.log.gz 2026-02-19T21:12:56.683 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123353.log 2026-02-19T21:12:56.695 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126736.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.3.83360.log.gz 2026-02-19T21:12:56.705 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100747.log 2026-02-19T21:12:56.706 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123353.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123353.log.gz 2026-02-19T21:12:56.706 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18951.log 2026-02-19T21:12:56.707 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100747.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100747.log.gz 2026-02-19T21:12:56.707 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76706.log 2026-02-19T21:12:56.708 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18951.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18951.log.gz 2026-02-19T21:12:56.708 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44989.log 2026-02-19T21:12:56.712 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76706.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.76706.log.gz 2026-02-19T21:12:56.712 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78669.log 2026-02-19T21:12:56.712 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44989.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107946.log 2026-02-19T21:12:56.714 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51261.log.gz 2026-02-19T21:12:56.715 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72776.log 2026-02-19T21:12:56.715 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107946.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107946.log.gz 2026-02-19T21:12:56.716 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76084.log 2026-02-19T21:12:56.719 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72776.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72776.log.gz 2026-02-19T21:12:56.720 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22623.log 2026-02-19T21:12:56.721 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76084.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76084.log.gz 2026-02-19T21:12:56.721 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127627.log 2026-02-19T21:12:56.722 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22623.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22623.log.gz 2026-02-19T21:12:56.722 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46504.log 2026-02-19T21:12:56.723 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127627.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.127627.log.gz 2026-02-19T21:12:56.723 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108402.log 2026-02-19T21:12:56.724 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46504.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46504.log.gz 2026-02-19T21:12:56.724 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125985.log 2026-02-19T21:12:56.724 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.108402.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.108402.log.gz 2026-02-19T21:12:56.725 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114009.log 2026-02-19T21:12:56.726 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125985.log: /var/log/ceph/cluster2-client.mirror.78669.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125985.log.gz 2026-02-19T21:12:56.727 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38284.log 2026-02-19T21:12:56.727 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114009.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114009.log.gz 2026-02-19T21:12:56.727 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19443.log 2026-02-19T21:12:56.728 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38284.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38284.log.gz 2026-02-19T21:12:56.728 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48085.log 2026-02-19T21:12:56.741 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.19443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82975.log 2026-02-19T21:12:56.742 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48085.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48085.log.gz 2026-02-19T21:12:56.749 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128366.log 2026-02-19T21:12:56.750 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82975.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82975.log.gz 2026-02-19T21:12:56.773 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42669.log 2026-02-19T21:12:56.774 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.128366.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.128366.log.gz 2026-02-19T21:12:56.789 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29103.log 2026-02-19T21:12:56.790 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42669.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78186.log.gz 2026-02-19T21:12:56.791 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42669.log.gz 2026-02-19T21:12:56.797 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86863.log.gz 2026-02-19T21:12:56.797 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37494.log 2026-02-19T21:12:56.798 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29103.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29103.log.gz 2026-02-19T21:12:56.798 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72641.log 2026-02-19T21:12:56.799 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37494.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37494.log.gz 2026-02-19T21:12:56.799 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27124.log 2026-02-19T21:12:56.799 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72641.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.72641.log.gz 2026-02-19T21:12:56.800 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85408.log 2026-02-19T21:12:56.801 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27124.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27124.log.gz 2026-02-19T21:12:56.801 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103087.log 2026-02-19T21:12:56.801 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85408.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85408.log.gz 2026-02-19T21:12:56.802 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49932.log 2026-02-19T21:12:56.803 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103087.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.103087.log.gz 2026-02-19T21:12:56.803 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81182.log 2026-02-19T21:12:56.804 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49932.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49932.log.gz 2026-02-19T21:12:56.804 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51699.log 2026-02-19T21:12:56.805 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81182.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81182.log.gz 2026-02-19T21:12:56.805 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26286.log 2026-02-19T21:12:56.805 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51699.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51699.log.gz 2026-02-19T21:12:56.806 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75499.log 2026-02-19T21:12:56.806 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59284.log 2026-02-19T21:12:56.806 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26286.log.gz 2026-02-19T21:12:56.807 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75499.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75499.log.gz 2026-02-19T21:12:56.808 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123713.log 2026-02-19T21:12:56.808 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59284.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59284.log.gz 2026-02-19T21:12:56.808 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97048.log 2026-02-19T21:12:56.809 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123713.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123713.log.gz 2026-02-19T21:12:56.809 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63009.log 2026-02-19T21:12:56.821 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49253.log 2026-02-19T21:12:56.822 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63009.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63009.log.gz 2026-02-19T21:12:56.829 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76049.log 2026-02-19T21:12:56.830 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49253.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49253.log.gz 2026-02-19T21:12:56.837 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34221.log 2026-02-19T21:12:56.838 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76049.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76049.log.gz 2026-02-19T21:12:56.845 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94730.log 2026-02-19T21:12:56.853 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34221.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126710.log 2026-02-19T21:12:56.854 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94730.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94730.log.gz 2026-02-19T21:12:56.857 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34221.log.gz 2026-02-19T21:12:56.861 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114868.log 2026-02-19T21:12:56.862 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126710.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126710.log.gz 2026-02-19T21:12:56.864 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.118389.log.gz 2026-02-19T21:12:56.864 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71508.log 2026-02-19T21:12:56.865 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114868.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114868.log.gz 2026-02-19T21:12:56.865 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55245.log 2026-02-19T21:12:56.866 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71508.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71508.log.gz 2026-02-19T21:12:56.866 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63071.log 2026-02-19T21:12:56.866 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55245.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55245.log.gz 2026-02-19T21:12:56.867 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88304.log 2026-02-19T21:12:56.867 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63071.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63071.log.gz 2026-02-19T21:12:56.868 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111037.log 2026-02-19T21:12:56.868 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.88304.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.88304.log.gz 2026-02-19T21:12:56.869 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30944.log 2026-02-19T21:12:56.881 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111037.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108427.log 2026-02-19T21:12:56.889 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30944.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81951.log.gz 2026-02-19T21:12:56.889 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19713.log 2026-02-19T21:12:56.891 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108427.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.108427.log.gz 2026-02-19T21:12:56.896 INFO:teuthology.orchestra.run.trial151.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.mirror.19443.log.gz 2026-02-19T21:12:56.897 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120313.log 2026-02-19T21:12:56.898 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.19713.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19713.log.gz 2026-02-19T21:12:56.898 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29650.log 2026-02-19T21:12:56.899 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120313.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120313.log.gz 2026-02-19T21:12:56.900 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120406.log 2026-02-19T21:12:56.900 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29650.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29650.log.gz 2026-02-19T21:12:56.901 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69604.log 2026-02-19T21:12:56.901 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120406.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120406.log.gz 2026-02-19T21:12:56.902 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98327.log 2026-02-19T21:12:56.905 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.69604.log: 95.8% 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.69604.log.gz 2026-02-19T21:12:56.905 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.83331.log.gz 2026-02-19T21:12:56.906 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59610.log 2026-02-19T21:12:56.906 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30410.log 2026-02-19T21:12:56.906 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98327.log: /var/log/ceph/cluster2-client.mirror.59610.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59610.log.gz 2026-02-19T21:12:56.908 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127694.log 2026-02-19T21:12:56.908 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30410.log: 92.2%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90801.log 2026-02-19T21:12:56.908 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.98327.log.gz 2026-02-19T21:12:56.908 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30410.log.gz 2026-02-19T21:12:56.908 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59707.log 2026-02-19T21:12:56.909 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127694.log: /var/log/ceph/cluster1-client.mirror.90801.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90801.log.gz 2026-02-19T21:12:56.910 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123443.log 2026-02-19T21:12:56.911 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59707.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59707.log.gz 2026-02-19T21:12:56.911 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37805.log 2026-02-19T21:12:56.912 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123443.log: 84.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123443.log.gz 2026-02-19T21:12:56.912 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110064.log 2026-02-19T21:12:56.913 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37805.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33930.log 2026-02-19T21:12:56.913 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37805.log.gz 2026-02-19T21:12:56.914 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110064.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110064.log.gz 2026-02-19T21:12:56.915 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105930.log 2026-02-19T21:12:56.916 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33930.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33930.log.gz 2026-02-19T21:12:56.916 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107472.log 2026-02-19T21:12:56.917 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105930.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105930.log.gz 2026-02-19T21:12:56.917 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74554.log 2026-02-19T21:12:56.918 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.107472.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.107472.log.gz 2026-02-19T21:12:56.918 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60743.log 2026-02-19T21:12:56.919 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74554.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74554.log.gz 2026-02-19T21:12:56.919 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19556.log 2026-02-19T21:12:56.920 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60743.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60743.log.gz 2026-02-19T21:12:56.920 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100060.log 2026-02-19T21:12:56.921 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19556.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47115.log 2026-02-19T21:12:56.921 INFO:teuthology.orchestra.run.trial151.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.19556.log.gz 2026-02-19T21:12:56.921 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100060.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100060.log.gz 2026-02-19T21:12:56.922 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125168.log 2026-02-19T21:12:56.923 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47115.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47115.log.gz 2026-02-19T21:12:56.923 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47565.log 2026-02-19T21:12:56.924 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125168.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125168.log.gz 2026-02-19T21:12:56.924 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58314.log 2026-02-19T21:12:56.924 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47565.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47565.log.gz 2026-02-19T21:12:56.925 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21449.log 2026-02-19T21:12:56.925 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58314.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58314.log.gz 2026-02-19T21:12:56.926 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34353.log 2026-02-19T21:12:56.926 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21449.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75887.log 2026-02-19T21:12:56.926 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.21449.log.gz 2026-02-19T21:12:56.927 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34353.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34353.log.gz 2026-02-19T21:12:56.928 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73293.log 2026-02-19T21:12:56.929 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75887.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75887.log.gz 2026-02-19T21:12:56.929 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25378.log 2026-02-19T21:12:56.929 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73293.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73293.log.gz 2026-02-19T21:12:56.929 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27935.log 2026-02-19T21:12:56.930 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25378.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.25378.log.gz 2026-02-19T21:12:56.930 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45437.log 2026-02-19T21:12:56.931 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27935.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27935.log.gz 2026-02-19T21:12:56.931 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38901.log 2026-02-19T21:12:56.932 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45437.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45437.log.gz 2026-02-19T21:12:56.932 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121623.log 2026-02-19T21:12:56.933 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38901.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38901.log.gz 2026-02-19T21:12:56.933 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119429.log 2026-02-19T21:12:56.934 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121623.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121623.log.gz 2026-02-19T21:12:56.935 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119948.log 2026-02-19T21:12:56.935 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119429.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.119429.log.gz 2026-02-19T21:12:56.936 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55439.log 2026-02-19T21:12:56.937 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119948.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.119948.log.gz 2026-02-19T21:12:56.937 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125060.log 2026-02-19T21:12:56.937 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55439.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55439.log.gz 2026-02-19T21:12:56.938 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21972.log 2026-02-19T21:12:56.938 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125060.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.125060.log.gz 2026-02-19T21:12:56.939 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109732.log 2026-02-19T21:12:56.939 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21972.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21972.log.gz 2026-02-19T21:12:56.940 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63653.log 2026-02-19T21:12:56.940 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109732.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109732.log.gz 2026-02-19T21:12:56.940 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28676.log 2026-02-19T21:12:56.941 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63653.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63653.log.gz 2026-02-19T21:12:56.941 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97220.log 2026-02-19T21:12:56.942 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28676.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84340.log 2026-02-19T21:12:56.942 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28676.log.gz 2026-02-19T21:12:56.943 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97220.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97220.log.gz 2026-02-19T21:12:56.943 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27318.log 2026-02-19T21:12:56.946 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.84340.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84340.log.gz 2026-02-19T21:12:56.946 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113277.log 2026-02-19T21:12:56.947 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27318.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27318.log.gz 2026-02-19T21:12:56.947 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103958.log 2026-02-19T21:12:56.948 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113277.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113277.log.gz 2026-02-19T21:12:56.948 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111129.log 2026-02-19T21:12:56.948 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103958.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103958.log.gz 2026-02-19T21:12:56.949 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37378.log 2026-02-19T21:12:56.949 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111129.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111129.log.gz 2026-02-19T21:12:56.950 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66580.log 2026-02-19T21:12:56.951 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37378.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37378.log.gz 2026-02-19T21:12:56.951 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76211.log 2026-02-19T21:12:56.951 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66580.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66580.log.gz 2026-02-19T21:12:56.952 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120739.log 2026-02-19T21:12:56.952 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76211.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76211.log.gz 2026-02-19T21:12:56.953 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65804.log 2026-02-19T21:12:56.954 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120739.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120739.log.gz 2026-02-19T21:12:56.954 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45728.log 2026-02-19T21:12:56.955 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65804.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65804.log.gz 2026-02-19T21:12:56.955 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127429.log 2026-02-19T21:12:56.956 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45728.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45728.log.gz 2026-02-19T21:12:56.956 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90006.log 2026-02-19T21:12:56.956 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.127429.log: 85.3% -- replaced with /var/log/ceph/cluster1-client.mirror.127429.log.gz 2026-02-19T21:12:56.957 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48438.log 2026-02-19T21:12:56.957 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90006.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.90006.log.gz 2026-02-19T21:12:56.958 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.38094.log 2026-02-19T21:12:56.958 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48438.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48438.log.gz 2026-02-19T21:12:56.959 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96510.log 2026-02-19T21:12:56.959 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.3.38094.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24990.log 2026-02-19T21:12:56.969 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39483.log 2026-02-19T21:12:56.970 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96510.log: /var/log/ceph/cluster2-client.mirror.24990.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96510.log.gz 2026-02-19T21:12:56.977 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100450.log 2026-02-19T21:12:56.981 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39483.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24990.log.gz 2026-02-19T21:12:56.989 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.114684.log.gz 2026-02-19T21:12:56.989 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31449.log 2026-02-19T21:12:56.989 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39483.log.gz 2026-02-19T21:12:57.005 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100450.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76882.log 2026-02-19T21:12:57.006 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31449.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.31449.log.gz 2026-02-19T21:12:57.013 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103337.log 2026-02-19T21:12:57.029 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76882.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76882.log.gz 2026-02-19T21:12:57.033 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41233.log 2026-02-19T21:12:57.034 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103337.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103337.log.gz 2026-02-19T21:12:57.042 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33639.log 2026-02-19T21:12:57.049 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90041.log 2026-02-19T21:12:57.054 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33639.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41233.log.gz 2026-02-19T21:12:57.057 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33639.log.gz 2026-02-19T21:12:57.061 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75605.log 2026-02-19T21:12:57.062 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90041.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90041.log.gz 2026-02-19T21:12:57.069 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.111037.log.gz 2026-02-19T21:12:57.069 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108447.log 2026-02-19T21:12:57.073 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75605.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75605.log.gz 2026-02-19T21:12:57.073 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60254.log 2026-02-19T21:12:57.074 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.108447.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.108447.log.gz 2026-02-19T21:12:57.075 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104402.log 2026-02-19T21:12:57.075 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60254.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60254.log.gz 2026-02-19T21:12:57.076 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66968.log 2026-02-19T21:12:57.076 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104402.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104402.log.gz 2026-02-19T21:12:57.077 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120568.log 2026-02-19T21:12:57.077 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66968.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66968.log.gz 2026-02-19T21:12:57.077 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67259.log 2026-02-19T21:12:57.078 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120568.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.120568.log.gz 2026-02-19T21:12:57.079 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21693.log 2026-02-19T21:12:57.079 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67259.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67259.log.gz 2026-02-19T21:12:57.080 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123016.log 2026-02-19T21:12:57.080 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87528.log.gz 2026-02-19T21:12:57.080 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21693.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.21693.log.gz 2026-02-19T21:12:57.082 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72447.log 2026-02-19T21:12:57.082 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123016.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119245.log 2026-02-19T21:12:57.082 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123016.log.gz 2026-02-19T21:12:57.082 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72447.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36196.log 2026-02-19T21:12:57.083 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119245.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72447.log.gz 2026-02-19T21:12:57.083 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119245.log.gz 2026-02-19T21:12:57.083 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67453.log 2026-02-19T21:12:57.084 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29844.log 2026-02-19T21:12:57.084 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36196.log: /var/log/ceph/cluster2-client.mirror.67453.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67453.log.gz 2026-02-19T21:12:57.084 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36196.log.gz 2026-02-19T21:12:57.084 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27609.log 2026-02-19T21:12:57.085 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84267.log 2026-02-19T21:12:57.085 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29844.log: /var/log/ceph/cluster2-client.mirror.27609.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29844.log.gz 2026-02-19T21:12:57.086 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27609.log.gz 2026-02-19T21:12:57.086 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126248.log 2026-02-19T21:12:57.086 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49606.log 2026-02-19T21:12:57.086 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.84267.log: /var/log/ceph/cluster2-client.mirror.126248.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126248.log.gz 2026-02-19T21:12:57.086 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106301.log 2026-02-19T21:12:57.087 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49606.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49606.log.gz 2026-02-19T21:12:57.087 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119054.log 2026-02-19T21:12:57.087 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106301.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106301.log.gz 2026-02-19T21:12:57.088 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39875.log 2026-02-19T21:12:57.088 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119054.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119054.log.gz 2026-02-19T21:12:57.089 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111565.log 2026-02-19T21:12:57.089 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39875.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39875.log.gz 2026-02-19T21:12:57.089 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47953.log 2026-02-19T21:12:57.090 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111565.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111565.log.gz 2026-02-19T21:12:57.090 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102153.log 2026-02-19T21:12:57.090 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47953.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47953.log.gz 2026-02-19T21:12:57.091 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110106.log 2026-02-19T21:12:57.091 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102153.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.102153.log.gz 2026-02-19T21:12:57.092 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46213.log 2026-02-19T21:12:57.106 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110106.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99584.log 2026-02-19T21:12:57.106 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46213.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46213.log.gz 2026-02-19T21:12:57.115 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126736.log.gz 2026-02-19T21:12:57.117 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46636.log 2026-02-19T21:12:57.118 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99584.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99584.log.gz 2026-02-19T21:12:57.119 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87057.log 2026-02-19T21:12:57.119 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46636.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46636.log.gz 2026-02-19T21:12:57.120 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62330.log 2026-02-19T21:12:57.120 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87057.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87057.log.gz 2026-02-19T21:12:57.120 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122396.log 2026-02-19T21:12:57.121 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62330.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62330.log.gz 2026-02-19T21:12:57.121 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62198.log 2026-02-19T21:12:57.122 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122396.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.122396.log.gz 2026-02-19T21:12:57.123 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72350.log 2026-02-19T21:12:57.123 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62198.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62198.log.gz 2026-02-19T21:12:57.123 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108979.log 2026-02-19T21:12:57.125 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72350.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72350.log.gz 2026-02-19T21:12:57.125 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19479.log 2026-02-19T21:12:57.126 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.108979.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.108979.log.gz 2026-02-19T21:12:57.127 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101363.log 2026-02-19T21:12:57.127 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.19479.log: 78.2% -- replaced with /var/log/ceph/cluster2-client.mirror.19479.log.gz 2026-02-19T21:12:57.127 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120090.log 2026-02-19T21:12:57.128 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101363.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101363.log.gz 2026-02-19T21:12:57.128 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73778.log 2026-02-19T21:12:57.128 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120090.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.120090.log.gz 2026-02-19T21:12:57.129 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79733.log 2026-02-19T21:12:57.129 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73778.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73778.log.gz 2026-02-19T21:12:57.129 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90875.log 2026-02-19T21:12:57.130 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79733.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79733.log.gz 2026-02-19T21:12:57.130 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44595.log 2026-02-19T21:12:57.131 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90875.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.90875.log.gz 2026-02-19T21:12:57.131 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86729.log 2026-02-19T21:12:57.132 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44595.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44595.log.gz 2026-02-19T21:12:57.132 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120355.log 2026-02-19T21:12:57.133 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86729.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86729.log.gz 2026-02-19T21:12:57.133 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81928.log 2026-02-19T21:12:57.135 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120355.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.120355.log.gz 2026-02-19T21:12:57.135 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32516.log 2026-02-19T21:12:57.136 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81928.log: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81928.log.gz 2026-02-19T21:12:57.136 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51734.log 2026-02-19T21:12:57.137 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32516.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32516.log.gz 2026-02-19T21:12:57.137 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61616.log 2026-02-19T21:12:57.139 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51734.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51734.log.gz 2026-02-19T21:12:57.140 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18794.log 2026-02-19T21:12:57.140 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61616.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61616.log.gz 2026-02-19T21:12:57.140 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52902.log 2026-02-19T21:12:57.140 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18794.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18794.log.gz 2026-02-19T21:12:57.140 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57767.log 2026-02-19T21:12:57.144 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52902.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52902.log.gz 2026-02-19T21:12:57.144 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27027.log 2026-02-19T21:12:57.144 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57767.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57767.log.gz 2026-02-19T21:12:57.144 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49738.log 2026-02-19T21:12:57.144 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27027.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.27027.log.gz 2026-02-19T21:12:57.144 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40325.log 2026-02-19T21:12:57.144 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49738.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49738.log.gz 2026-02-19T21:12:57.145 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90689.log 2026-02-19T21:12:57.151 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40325.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40325.log.gz 2026-02-19T21:12:57.151 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75252.log 2026-02-19T21:12:57.151 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90689.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.90689.log.gz 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18735.log 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75252.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75252.log.gz 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88945.log.gz 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123601.log 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18735.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18735.log.gz 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94306.log 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91169.log 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123601.log: /var/log/ceph/cluster2-client.mirror.94306.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123601.log.gz 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94306.log.gz 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54822.log 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28420.log 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.91169.log: /var/log/ceph/cluster2-client.mirror.54822.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.91169.log.gz 2026-02-19T21:12:57.152 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54822.log.gz 2026-02-19T21:12:57.153 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39192.log 2026-02-19T21:12:57.153 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100712.log 2026-02-19T21:12:57.153 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28420.log: /var/log/ceph/cluster2-client.mirror.39192.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28420.log.gz 2026-02-19T21:12:57.153 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123938.log 2026-02-19T21:12:57.153 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39192.log.gz 2026-02-19T21:12:57.153 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100712.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100712.log.gz 2026-02-19T21:12:57.153 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23216.log 2026-02-19T21:12:57.153 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126816.log 2026-02-19T21:12:57.153 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123938.log: /var/log/ceph/cluster2-client.mirror.23216.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123938.log.gz 2026-02-19T21:12:57.154 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87506.log 2026-02-19T21:12:57.155 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23216.log.gz 2026-02-19T21:12:57.155 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126816.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126816.log.gz 2026-02-19T21:12:57.155 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128762.log 2026-02-19T21:12:57.155 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39254.log 2026-02-19T21:12:57.155 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.87506.log: /var/log/ceph/cluster2-client.admin.128762.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128762.log.gz 2026-02-19T21:12:57.156 INFO:teuthology.orchestra.run.trial151.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.87506.log.gz 2026-02-19T21:12:57.156 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.19556.log 2026-02-19T21:12:57.156 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33092.log 2026-02-19T21:12:57.156 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39254.log: /var/log/ceph/cluster2-client.mirror.2.19556.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67418.log 2026-02-19T21:12:57.156 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39254.log.gz 2026-02-19T21:12:57.157 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33092.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33092.log.gz 2026-02-19T21:12:57.157 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37594.log 2026-02-19T21:12:57.158 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67418.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67418.log.gz 2026-02-19T21:12:57.158 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60355.log 2026-02-19T21:12:57.158 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37594.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88765.log 2026-02-19T21:12:57.162 INFO:teuthology.orchestra.run.trial151.stderr: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.2.19556.log.gz 2026-02-19T21:12:57.162 INFO:teuthology.orchestra.run.trial151.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37594.log.gz 2026-02-19T21:12:57.163 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60355.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60355.log.gz 2026-02-19T21:12:57.163 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74263.log 2026-02-19T21:12:57.164 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73196.log 2026-02-19T21:12:57.164 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88765.log: /var/log/ceph/cluster2-client.mirror.74263.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88765.log.gz 2026-02-19T21:12:57.164 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74263.log.gz 2026-02-19T21:12:57.165 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96781.log 2026-02-19T21:12:57.165 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116518.log 2026-02-19T21:12:57.165 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73196.log: /var/log/ceph/cluster2-client.mirror.96781.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73196.log.gz 2026-02-19T21:12:57.165 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96781.log.gz 2026-02-19T21:12:57.165 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30232.log 2026-02-19T21:12:57.166 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57476.log 2026-02-19T21:12:57.166 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116518.log: /var/log/ceph/cluster2-client.mirror.30232.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.116518.log.gz 2026-02-19T21:12:57.167 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63397.log 2026-02-19T21:12:57.167 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30232.log.gz 2026-02-19T21:12:57.167 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57476.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57476.log.gz 2026-02-19T21:12:57.167 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77459.log 2026-02-19T21:12:57.168 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44944.log 2026-02-19T21:12:57.168 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63397.log: /var/log/ceph/cluster2-client.mirror.77459.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63397.log.gz 2026-02-19T21:12:57.168 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122015.log 2026-02-19T21:12:57.168 INFO:teuthology.orchestra.run.trial151.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77459.log.gz 2026-02-19T21:12:57.169 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.44944.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123151.log 2026-02-19T21:12:57.172 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.44944.log.gz 2026-02-19T21:12:57.173 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122015.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.122015.log.gz 2026-02-19T21:12:57.174 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109243.log 2026-02-19T21:12:57.174 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.19427.log 2026-02-19T21:12:57.175 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109243.log: /var/log/ceph/cluster1-client.mirror.123151.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123151.log.gz 2026-02-19T21:12:57.175 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109243.log.gz 2026-02-19T21:12:57.175 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126390.log 2026-02-19T21:12:57.176 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56374.log 2026-02-19T21:12:57.176 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.19427.log: /var/log/ceph/cluster1-client.mirror.126390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68776.log 2026-02-19T21:12:57.176 INFO:teuthology.orchestra.run.trial151.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.126390.log.gz 2026-02-19T21:12:57.176 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92626.log 2026-02-19T21:12:57.177 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68776.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68776.log.gz 2026-02-19T21:12:57.180 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78941.log.gz 2026-02-19T21:12:57.181 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56374.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56374.log.gz 2026-02-19T21:12:57.193 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69102.log 2026-02-19T21:12:57.195 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92626.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37659.log 2026-02-19T21:12:57.195 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.92626.log.gz/var/log/ceph/cluster2-client.mirror.69102.log: 2026-02-19T21:12:57.195 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69102.log.gz 2026-02-19T21:12:57.195 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69326.log 2026-02-19T21:12:57.195 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37659.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60487.log 2026-02-19T21:12:57.197 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69326.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69326.log.gz 2026-02-19T21:12:57.197 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128517.log 2026-02-19T21:12:57.197 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60487.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60487.log.gz 2026-02-19T21:12:57.197 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123286.log 2026-02-19T21:12:57.198 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.128517.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.128517.log.gz 2026-02-19T21:12:57.198 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85861.log 2026-02-19T21:12:57.198 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123286.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123286.log.gz 2026-02-19T21:12:57.199 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62877.log 2026-02-19T21:12:57.199 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85861.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85861.log.gz 2026-02-19T21:12:57.200 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77480.log 2026-02-19T21:12:57.200 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62877.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62877.log.gz 2026-02-19T21:12:57.201 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63847.log 2026-02-19T21:12:57.201 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77480.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77480.log.gz 2026-02-19T21:12:57.202 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57829.log 2026-02-19T21:12:57.202 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63847.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63847.log.gz 2026-02-19T21:12:57.203 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128416.log 2026-02-19T21:12:57.203 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57829.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57829.log.gz 2026-02-19T21:12:57.204 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121558.log 2026-02-19T21:12:57.204 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.128416.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.128416.log.gz 2026-02-19T21:12:57.204 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93902.log 2026-02-19T21:12:57.206 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121558.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121558.log.gz 2026-02-19T21:12:57.206 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102728.log 2026-02-19T21:12:57.207 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93902.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93902.log.gz 2026-02-19T21:12:57.207 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58834.log 2026-02-19T21:12:57.208 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102728.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102728.log.gz 2026-02-19T21:12:57.208 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28812.log 2026-02-19T21:12:57.209 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58834.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58834.log.gz 2026-02-19T21:12:57.209 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83793.log.gz 2026-02-19T21:12:57.209 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19553.log 2026-02-19T21:12:57.210 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28812.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28812.log.gz 2026-02-19T21:12:57.210 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41427.log 2026-02-19T21:12:57.210 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83874.log 2026-02-19T21:12:57.210 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19553.log: /var/log/ceph/cluster2-client.mirror.41427.log: 78.0% -- replaced with /var/log/ceph/cluster1-client.mirror.19553.log.gz 2026-02-19T21:12:57.211 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31740.log 2026-02-19T21:12:57.211 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41427.log.gz 2026-02-19T21:12:57.212 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83874.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83874.log.gz 2026-02-19T21:12:57.212 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56506.log 2026-02-19T21:12:57.212 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61360.log 2026-02-19T21:12:57.213 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31740.log: /var/log/ceph/cluster2-client.mirror.56506.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56506.log.gz 2026-02-19T21:12:57.213 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31740.log.gz 2026-02-19T21:12:57.213 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127976.log 2026-02-19T21:12:57.213 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95379.log 2026-02-19T21:12:57.213 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61360.log: /var/log/ceph/cluster2-client.mirror.127976.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61360.log.gz 2026-02-19T21:12:57.214 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124726.log 2026-02-19T21:12:57.214 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95379.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95379.log.gz 2026-02-19T21:12:57.214 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-19T21:12:57.215 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124726.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124726.log.gz 2026-02-19T21:12:57.215 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84317.log 2026-02-19T21:12:57.216 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113678.log 2026-02-19T21:12:57.216 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.78669.log.gz 2026-02-19T21:12:57.216 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106398.log 2026-02-19T21:12:57.217 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113678.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.113678.log.gz 2026-02-19T21:12:57.218 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109181.log 2026-02-19T21:12:57.218 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106398.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.106398.log.gz 2026-02-19T21:12:57.218 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35420.log 2026-02-19T21:12:57.219 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109181.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109181.log.gz 2026-02-19T21:12:57.219 INFO:teuthology.orchestra.run.trial151.stderr: 92.9% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-19T21:12:57.220 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20078.log 2026-02-19T21:12:57.220 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.84317.log: /var/log/ceph/cluster2-client.mirror.35420.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35420.log.gz 2026-02-19T21:12:57.221 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77140.log 2026-02-19T21:12:57.221 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20078.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.20078.log.gz 2026-02-19T21:12:57.221 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50983.log 2026-02-19T21:12:57.225 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.77140.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.77140.log.gz 2026-02-19T21:12:57.225 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33833.log 2026-02-19T21:12:57.226 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50983.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50983.log.gz 2026-02-19T21:12:57.226 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86644.log 2026-02-19T21:12:57.227 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33833.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110938.log 2026-02-19T21:12:57.227 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33833.log.gz 2026-02-19T21:12:57.228 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86644.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.86644.log.gz 2026-02-19T21:12:57.228 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68423.log 2026-02-19T21:12:57.229 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110938.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110938.log.gz 2026-02-19T21:12:57.229 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121652.log 2026-02-19T21:12:57.230 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68423.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68423.log.gz 2026-02-19T21:12:57.230 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59381.log 2026-02-19T21:12:57.231 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121652.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.121652.log.gz 2026-02-19T21:12:57.231 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111334.log 2026-02-19T21:12:57.232 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59381.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59381.log.gz 2026-02-19T21:12:57.232 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87876.log 2026-02-19T21:12:57.233 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111334.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.111334.log.gz 2026-02-19T21:12:57.233 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58508.log 2026-02-19T21:12:57.234 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87876.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.87876.log.gz 2026-02-19T21:12:57.234 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125270.log 2026-02-19T21:12:57.235 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58508.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58508.log.gz 2026-02-19T21:12:57.235 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50576.log 2026-02-19T21:12:57.236 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125270.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.125270.log.gz 2026-02-19T21:12:57.236 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124947.log 2026-02-19T21:12:57.237 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50576.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50576.log.gz 2026-02-19T21:12:57.237 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83575.log 2026-02-19T21:12:57.238 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.124947.log: 63.8% -- replaced with /var/log/ceph/cluster2-client.mirror.124947.log.gz 2026-02-19T21:12:57.238 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31220.log 2026-02-19T21:12:57.242 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83575.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125032.log.gz 2026-02-19T21:12:57.242 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92289.log 2026-02-19T21:12:57.243 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31220.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31220.log.gz 2026-02-19T21:12:57.257 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96360.log 2026-02-19T21:12:57.258 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92289.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.92289.log.gz 2026-02-19T21:12:57.258 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126330.log 2026-02-19T21:12:57.260 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96360.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.96360.log.gz 2026-02-19T21:12:57.260 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111164.log 2026-02-19T21:12:57.261 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126330.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126330.log.gz 2026-02-19T21:12:57.261 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98270.log 2026-02-19T21:12:57.262 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111164.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.111164.log.gz 2026-02-19T21:12:57.262 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86233.log 2026-02-19T21:12:57.263 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98270.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98270.log.gz 2026-02-19T21:12:57.263 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45631.log 2026-02-19T21:12:57.264 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86233.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.86233.log.gz 2026-02-19T21:12:57.264 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109651.log 2026-02-19T21:12:57.265 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45631.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45631.log.gz 2026-02-19T21:12:57.265 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59416.log 2026-02-19T21:12:57.266 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109651.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109651.log.gz 2026-02-19T21:12:57.266 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68097.log 2026-02-19T21:12:57.267 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59416.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59416.log.gz 2026-02-19T21:12:57.267 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70538.log 2026-02-19T21:12:57.268 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68097.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68097.log.gz 2026-02-19T21:12:57.268 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50029.log 2026-02-19T21:12:57.269 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70538.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70538.log.gz 2026-02-19T21:12:57.269 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18950.log 2026-02-19T21:12:57.270 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50029.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.50029.log.gz 2026-02-19T21:12:57.270 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29976.log 2026-02-19T21:12:57.270 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18950.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18950.log.gz 2026-02-19T21:12:57.271 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73099.log 2026-02-19T21:12:57.272 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29976.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29976.log.gz 2026-02-19T21:12:57.272 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65575.log 2026-02-19T21:12:57.272 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73099.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73099.log.gz 2026-02-19T21:12:57.273 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64041.log 2026-02-19T21:12:57.273 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65575.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65575.log.gz 2026-02-19T21:12:57.274 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25704.log 2026-02-19T21:12:57.274 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64041.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64041.log.gz 2026-02-19T21:12:57.275 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69005.log 2026-02-19T21:12:57.276 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25704.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25704.log.gz 2026-02-19T21:12:57.276 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111376.log 2026-02-19T21:12:57.276 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69005.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69005.log.gz 2026-02-19T21:12:57.277 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72090.log 2026-02-19T21:12:57.277 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111376.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.111376.log.gz 2026-02-19T21:12:57.278 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89449.log 2026-02-19T21:12:57.278 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72090.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72090.log.gz 2026-02-19T21:12:57.279 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58640.log 2026-02-19T21:12:57.280 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.89449.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.89449.log.gz 2026-02-19T21:12:57.280 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78646.log 2026-02-19T21:12:57.281 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58640.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58640.log.gz 2026-02-19T21:12:57.281 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87022.log 2026-02-19T21:12:57.282 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78646.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78646.log.gz 2026-02-19T21:12:57.282 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87908.log 2026-02-19T21:12:57.283 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.87022.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.87022.log.gz 2026-02-19T21:12:57.283 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121966.log 2026-02-19T21:12:57.284 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.87908.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87908.log.gz 2026-02-19T21:12:57.284 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88997.log 2026-02-19T21:12:57.285 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121966.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121966.log.gz 2026-02-19T21:12:57.285 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54434.log 2026-02-19T21:12:57.286 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88997.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88997.log.gz 2026-02-19T21:12:57.286 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23274.log 2026-02-19T21:12:57.287 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54434.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54434.log.gz 2026-02-19T21:12:57.287 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126628.log 2026-02-19T21:12:57.288 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23274.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23274.log.gz 2026-02-19T21:12:57.288 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110084.log 2026-02-19T21:12:57.289 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126628.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126628.log.gz 2026-02-19T21:12:57.289 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95909.log 2026-02-19T21:12:57.291 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.110084.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.110084.log.gz 2026-02-19T21:12:57.291 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43528.log 2026-02-19T21:12:57.292 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.95909.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.95909.log.gz 2026-02-19T21:12:57.292 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104783.log 2026-02-19T21:12:57.293 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43528.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43528.log.gz 2026-02-19T21:12:57.293 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25510.log 2026-02-19T21:12:57.294 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104783.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104783.log.gz 2026-02-19T21:12:57.294 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115382.log 2026-02-19T21:12:57.295 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25510.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25510.log.gz 2026-02-19T21:12:57.295 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49412.log 2026-02-19T21:12:57.296 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115382.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.115382.log.gz 2026-02-19T21:12:57.296 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97704.log 2026-02-19T21:12:57.297 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49412.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49412.log.gz 2026-02-19T21:12:57.297 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128541.log 2026-02-19T21:12:57.305 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97704.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68582.log 2026-02-19T21:12:57.306 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128541.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128541.log.gz 2026-02-19T21:12:57.321 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16018.log 2026-02-19T21:12:57.322 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68582.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68582.log.gz 2026-02-19T21:12:57.325 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.97048.log.gz 2026-02-19T21:12:57.326 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54857.log 2026-02-19T21:12:57.326 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.16018.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16018.log.gz 2026-02-19T21:12:57.327 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82357.log 2026-02-19T21:12:57.327 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54857.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54857.log.gz 2026-02-19T21:12:57.327 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26348.log 2026-02-19T21:12:57.341 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82357.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24311.log 2026-02-19T21:12:57.342 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26348.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26348.log.gz 2026-02-19T21:12:57.361 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23895.log 2026-02-19T21:12:57.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24311.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24311.log.gz 2026-02-19T21:12:57.377 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98008.log 2026-02-19T21:12:57.378 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23895.log: 78.2% -- replaced with /var/log/ceph/cluster2-client.mirror.23895.log.gz 2026-02-19T21:12:57.381 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.110106.log.gz 2026-02-19T21:12:57.381 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107216.log 2026-02-19T21:12:57.382 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98008.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98008.log.gz 2026-02-19T21:12:57.383 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100428.log 2026-02-19T21:12:57.383 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107216.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107216.log.gz 2026-02-19T21:12:57.383 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46539.log 2026-02-19T21:12:57.384 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.100428.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.100428.log.gz 2026-02-19T21:12:57.385 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64138.log 2026-02-19T21:12:57.385 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46539.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46539.log.gz 2026-02-19T21:12:57.386 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128032.log 2026-02-19T21:12:57.386 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64138.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64138.log.gz 2026-02-19T21:12:57.387 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53937.log 2026-02-19T21:12:57.387 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.128032.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.128032.log.gz 2026-02-19T21:12:57.388 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68194.log 2026-02-19T21:12:57.388 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.53937.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53937.log.gz 2026-02-19T21:12:57.388 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80337.log 2026-02-19T21:12:57.389 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68194.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68194.log.gz 2026-02-19T21:12:57.389 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65901.log 2026-02-19T21:12:57.395 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80337.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84267.log.gz 2026-02-19T21:12:57.397 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90781.log 2026-02-19T21:12:57.398 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65901.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65901.log.gz 2026-02-19T21:12:57.399 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48279.log 2026-02-19T21:12:57.400 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90781.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.90781.log.gz 2026-02-19T21:12:57.400 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65707.log 2026-02-19T21:12:57.401 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48279.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48279.log.gz 2026-02-19T21:12:57.401 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69962.log 2026-02-19T21:12:57.402 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65707.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65707.log.gz 2026-02-19T21:12:57.402 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19028.log 2026-02-19T21:12:57.421 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69962.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38998.log 2026-02-19T21:12:57.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19028.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19028.log.gz 2026-02-19T21:12:57.441 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76948.log 2026-02-19T21:12:57.443 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38998.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38998.log.gz 2026-02-19T21:12:57.453 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128840.log 2026-02-19T21:12:57.454 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76948.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81349.log.gz 2026-02-19T21:12:57.457 INFO:teuthology.orchestra.run.trial151.stderr: 93.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52964.log 2026-02-19T21:12:57.458 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128840.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128840.log.gz 2026-02-19T21:12:57.458 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.76948.log.gz 2026-02-19T21:12:57.459 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29262.log 2026-02-19T21:12:57.459 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52964.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52964.log.gz 2026-02-19T21:12:57.459 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94984.log 2026-02-19T21:12:57.460 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29262.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29262.log.gz 2026-02-19T21:12:57.460 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38672.log 2026-02-19T21:12:57.461 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.94984.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.94984.log.gz 2026-02-19T21:12:57.462 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123061.log 2026-02-19T21:12:57.462 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38672.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38672.log.gz 2026-02-19T21:12:57.463 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36161.log 2026-02-19T21:12:57.463 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123061.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123061.log.gz 2026-02-19T21:12:57.463 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26189.log 2026-02-19T21:12:57.464 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36161.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36161.log.gz 2026-02-19T21:12:57.464 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48826.log 2026-02-19T21:12:57.464 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26189.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26189.log.gz 2026-02-19T21:12:57.465 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33057.log 2026-02-19T21:12:57.465 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48826.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48826.log.gz 2026-02-19T21:12:57.466 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41621.log 2026-02-19T21:12:57.466 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82662.log.gz 2026-02-19T21:12:57.467 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33057.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107517.log 2026-02-19T21:12:57.467 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33057.log.gz 2026-02-19T21:12:57.467 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37413.log 2026-02-19T21:12:57.468 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41621.log: /var/log/ceph/cluster1-client.mirror.107517.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41621.log.gz 2026-02-19T21:12:57.468 INFO:teuthology.orchestra.run.trial151.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.107517.log.gz 2026-02-19T21:12:57.469 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47600.log 2026-02-19T21:12:57.469 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94238.log 2026-02-19T21:12:57.469 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37413.log: /var/log/ceph/cluster2-client.mirror.47600.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47600.log.gz 2026-02-19T21:12:57.470 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37413.log.gz 2026-02-19T21:12:57.470 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112575.log 2026-02-19T21:12:57.471 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30607.log 2026-02-19T21:12:57.471 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94238.log: /var/log/ceph/cluster2-client.mirror.112575.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112575.log.gz 2026-02-19T21:12:57.477 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100351.log 2026-02-19T21:12:57.478 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30607.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30607.log.gz 2026-02-19T21:12:57.493 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49315.log 2026-02-19T21:12:57.494 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100351.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100351.log.gz 2026-02-19T21:12:57.494 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126472.log 2026-02-19T21:12:57.495 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49315.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49315.log.gz 2026-02-19T21:12:57.495 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50708.log 2026-02-19T21:12:57.496 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126472.log: 82.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126472.log.gz 2026-02-19T21:12:57.496 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109778.log 2026-02-19T21:12:57.497 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50708.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50708.log.gz 2026-02-19T21:12:57.497 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19822.log 2026-02-19T21:12:57.498 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109778.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109778.log.gz 2026-02-19T21:12:57.498 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58058.log 2026-02-19T21:12:57.499 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85903.log 2026-02-19T21:12:57.499 INFO:teuthology.orchestra.run.trial151.stderr: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.19822.log.gz 2026-02-19T21:12:57.500 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58058.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58058.log.gz 2026-02-19T21:12:57.500 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107440.log 2026-02-19T21:12:57.501 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85903.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85903.log.gz 2026-02-19T21:12:57.501 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125900.log 2026-02-19T21:12:57.502 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107440.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107440.log.gz 2026-02-19T21:12:57.502 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44533.log 2026-02-19T21:12:57.502 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125900.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.125900.log.gz 2026-02-19T21:12:57.503 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105402.log 2026-02-19T21:12:57.504 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44533.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44533.log.gz 2026-02-19T21:12:57.504 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120019.log 2026-02-19T21:12:57.505 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105402.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.105402.log.gz 2026-02-19T21:12:57.505 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30572.log 2026-02-19T21:12:57.506 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120019.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120019.log.gz 2026-02-19T21:12:57.506 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123128.log 2026-02-19T21:12:57.507 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30572.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122993.log 2026-02-19T21:12:57.507 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30572.log.gz 2026-02-19T21:12:57.508 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123128.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123128.log.gz 2026-02-19T21:12:57.508 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59998.log 2026-02-19T21:12:57.508 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122993.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.122993.log.gz 2026-02-19T21:12:57.509 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125965.log 2026-02-19T21:12:57.509 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59998.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59998.log.gz 2026-02-19T21:12:57.510 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53905.log 2026-02-19T21:12:57.510 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125965.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125965.log.gz 2026-02-19T21:12:57.510 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127847.log 2026-02-19T21:12:57.511 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53905.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53905.log.gz 2026-02-19T21:12:57.511 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52025.log 2026-02-19T21:12:57.512 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127847.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127847.log.gz 2026-02-19T21:12:57.512 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55924.log 2026-02-19T21:12:57.513 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52025.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52025.log.gz 2026-02-19T21:12:57.513 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36972.log 2026-02-19T21:12:57.514 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55924.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55924.log.gz 2026-02-19T21:12:57.514 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77718.log 2026-02-19T21:12:57.515 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36972.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36972.log.gz 2026-02-19T21:12:57.515 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88684.log 2026-02-19T21:12:57.516 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.77718.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.77718.log.gz 2026-02-19T21:12:57.516 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33771.log 2026-02-19T21:12:57.517 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88684.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.88684.log.gz 2026-02-19T21:12:57.517 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122124.log 2026-02-19T21:12:57.518 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33771.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33771.log.gz 2026-02-19T21:12:57.518 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117821.log 2026-02-19T21:12:57.519 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122124.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122124.log.gz 2026-02-19T21:12:57.519 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122903.log 2026-02-19T21:12:57.520 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117821.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117821.log.gz 2026-02-19T21:12:57.520 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123803.log 2026-02-19T21:12:57.521 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122903.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122903.log.gz 2026-02-19T21:12:57.521 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62427.log 2026-02-19T21:12:57.521 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123803.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123803.log.gz 2026-02-19T21:12:57.522 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83094.log 2026-02-19T21:12:57.522 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62427.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62427.log.gz 2026-02-19T21:12:57.523 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54663.log 2026-02-19T21:12:57.533 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.83094.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62974.log 2026-02-19T21:12:57.534 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54663.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54663.log.gz 2026-02-19T21:12:57.549 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108740.log 2026-02-19T21:12:57.550 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62974.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62974.log.gz 2026-02-19T21:12:57.557 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85620.log 2026-02-19T21:12:57.569 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108740.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76577.log 2026-02-19T21:12:57.570 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85620.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85620.log.gz 2026-02-19T21:12:57.585 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71314.log 2026-02-19T21:12:57.586 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76577.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79298.log.gz 2026-02-19T21:12:57.589 INFO:teuthology.orchestra.run.trial151.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76577.log.gz 2026-02-19T21:12:57.589 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.95037.log 2026-02-19T21:12:57.590 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71314.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71314.log.gz 2026-02-19T21:12:57.590 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82858.log 2026-02-19T21:12:57.605 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.95037.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26542.log 2026-02-19T21:12:57.606 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82858.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82858.log.gz 2026-02-19T21:12:57.613 INFO:teuthology.orchestra.run.trial151.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.95037.log.gz 2026-02-19T21:12:57.614 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90333.log 2026-02-19T21:12:57.614 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26542.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.26542.log.gz 2026-02-19T21:12:57.614 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103765.log 2026-02-19T21:12:57.615 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90333.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90333.log.gz 2026-02-19T21:12:57.615 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27221.log 2026-02-19T21:12:57.617 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103765.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.103765.log.gz 2026-02-19T21:12:57.617 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123871.log 2026-02-19T21:12:57.618 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27221.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.27221.log.gz 2026-02-19T21:12:57.618 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114196.log 2026-02-19T21:12:57.619 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123871.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123871.log.gz 2026-02-19T21:12:57.619 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57053.log 2026-02-19T21:12:57.620 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114196.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114196.log.gz 2026-02-19T21:12:57.620 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121260.log 2026-02-19T21:12:57.621 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57053.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57053.log.gz 2026-02-19T21:12:57.621 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113812.log 2026-02-19T21:12:57.622 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121260.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121260.log.gz 2026-02-19T21:12:57.622 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25219.log 2026-02-19T21:12:57.623 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113812.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.113812.log.gz 2026-02-19T21:12:57.623 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77054.log 2026-02-19T21:12:57.624 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25219.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25219.log.gz 2026-02-19T21:12:57.624 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122276.log 2026-02-19T21:12:57.628 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.77054.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.77054.log.gz 2026-02-19T21:12:57.628 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39743.log 2026-02-19T21:12:57.629 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122276.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122276.log.gz 2026-02-19T21:12:57.629 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87461.log 2026-02-19T21:12:57.630 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39743.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39743.log.gz 2026-02-19T21:12:57.630 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29297.log 2026-02-19T21:12:57.631 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.87461.log: 95.7% 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87461.log.gz 2026-02-19T21:12:57.631 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.100450.log.gz 2026-02-19T21:12:57.631 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85037.log 2026-02-19T21:12:57.632 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29297.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.29297.log.gz 2026-02-19T21:12:57.632 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126791.log 2026-02-19T21:12:57.633 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.85037.log: 5.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85037.log.gz 2026-02-19T21:12:57.633 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37999.log 2026-02-19T21:12:57.633 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.126791.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.126791.log.gz 2026-02-19T21:12:57.634 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108513.log 2026-02-19T21:12:57.634 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116476.log 2026-02-19T21:12:57.635 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37999.log.gz 2026-02-19T21:12:57.635 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108513.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108513.log.gz 2026-02-19T21:12:57.635 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106733.log 2026-02-19T21:12:57.636 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116476.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.116476.log.gz 2026-02-19T21:12:57.636 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78213.log 2026-02-19T21:12:57.637 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.106733.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.106733.log.gz 2026-02-19T21:12:57.638 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88878.log 2026-02-19T21:12:57.638 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78213.log: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78213.log.gz 2026-02-19T21:12:57.638 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29941.log 2026-02-19T21:12:57.639 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.88878.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.88878.log.gz 2026-02-19T21:12:57.639 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63265.log 2026-02-19T21:12:57.640 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29941.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29941.log.gz 2026-02-19T21:12:57.640 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103515.log 2026-02-19T21:12:57.641 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63265.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63265.log.gz 2026-02-19T21:12:57.641 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57732.log 2026-02-19T21:12:57.642 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103515.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.103515.log.gz 2026-02-19T21:12:57.642 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88923.log 2026-02-19T21:12:57.642 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57732.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57732.log.gz 2026-02-19T21:12:57.643 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74166.log 2026-02-19T21:12:57.644 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.88923.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88923.log.gz 2026-02-19T21:12:57.645 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102533.log 2026-02-19T21:12:57.645 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74166.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74166.log.gz 2026-02-19T21:12:57.645 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77011.log 2026-02-19T21:12:57.647 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.102533.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.102533.log.gz 2026-02-19T21:12:57.647 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45243.log 2026-02-19T21:12:57.651 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.77011.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.77011.log.gz 2026-02-19T21:12:57.651 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111699.log 2026-02-19T21:12:57.652 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45243.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45243.log.gz 2026-02-19T21:12:57.652 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35288.log 2026-02-19T21:12:57.653 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111699.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.111699.log.gz 2026-02-19T21:12:57.654 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50673.log 2026-02-19T21:12:57.654 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35288.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35288.log.gz 2026-02-19T21:12:57.655 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89032.log 2026-02-19T21:12:57.655 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50673.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50673.log.gz 2026-02-19T21:12:57.655 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84244.log 2026-02-19T21:12:57.656 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89032.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89032.log.gz 2026-02-19T21:12:57.656 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35323.log 2026-02-19T21:12:57.657 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.84244.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.84244.log.gz 2026-02-19T21:12:57.657 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43625.log 2026-02-19T21:12:57.658 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37069.log 2026-02-19T21:12:57.658 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35323.log.gz 2026-02-19T21:12:57.659 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43625.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43625.log.gz 2026-02-19T21:12:57.659 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110903.log 2026-02-19T21:12:57.660 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37069.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37069.log.gz 2026-02-19T21:12:57.660 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38094.log 2026-02-19T21:12:57.661 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110903.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110903.log.gz 2026-02-19T21:12:57.661 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45031.log 2026-02-19T21:12:57.669 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38094.log: 92.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31255.log 2026-02-19T21:12:57.670 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.45031.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.admin.45031.log.gz 2026-02-19T21:12:57.670 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.38094.log.gz 2026-02-19T21:12:57.670 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51540.log 2026-02-19T21:12:57.671 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31255.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85501.log 2026-02-19T21:12:57.671 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31255.log.gz 2026-02-19T21:12:57.672 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51540.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51540.log.gz 2026-02-19T21:12:57.672 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92173.log 2026-02-19T21:12:57.673 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85501.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.85501.log.gz 2026-02-19T21:12:57.673 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80314.log 2026-02-19T21:12:57.674 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92173.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.92173.log.gz 2026-02-19T21:12:57.674 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24602.log 2026-02-19T21:12:57.675 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80314.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80314.log.gz 2026-02-19T21:12:57.675 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82998.log 2026-02-19T21:12:57.675 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24602.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65769.log 2026-02-19T21:12:57.676 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24602.log.gz 2026-02-19T21:12:57.676 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.94238.log.gz/var/log/ceph/cluster2-client.mirror.82998.log: 2026-02-19T21:12:57.677 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57538.log 2026-02-19T21:12:57.677 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65769.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65769.log.gz 2026-02-19T21:12:57.677 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121844.log 2026-02-19T21:12:57.678 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57538.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57538.log.gz 2026-02-19T21:12:57.678 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94820.log 2026-02-19T21:12:57.680 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121844.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121844.log.gz 2026-02-19T21:12:57.693 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67806.log 2026-02-19T21:12:57.694 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94820.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94820.log.gz 2026-02-19T21:12:57.695 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123511.log 2026-02-19T21:12:57.695 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67806.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67806.log.gz 2026-02-19T21:12:57.695 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26639.log 2026-02-19T21:12:57.696 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123511.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123511.log.gz 2026-02-19T21:12:57.696 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99098.log 2026-02-19T21:12:57.697 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26639.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26639.log.gz 2026-02-19T21:12:57.697 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86821.log 2026-02-19T21:12:57.698 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99098.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.99098.log.gz 2026-02-19T21:12:57.698 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47759.log 2026-02-19T21:12:57.700 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86821.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86821.log.gz 2026-02-19T21:12:57.700 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122465.log 2026-02-19T21:12:57.701 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47759.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47759.log.gz 2026-02-19T21:12:57.701 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.82200.log 2026-02-19T21:12:57.702 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122465.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122465.log.gz 2026-02-19T21:12:57.702 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26868.log 2026-02-19T21:12:57.709 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.2.82200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101305.log 2026-02-19T21:12:57.717 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26868.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79802.log 2026-02-19T21:12:57.718 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101305.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26868.log.gz 2026-02-19T21:12:57.718 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101305.log.gz 2026-02-19T21:12:57.721 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127976.log.gz 2026-02-19T21:12:57.734 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30135.log 2026-02-19T21:12:57.734 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79802.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79802.log.gz 2026-02-19T21:12:57.734 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46054.log 2026-02-19T21:12:57.735 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30135.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30135.log.gz 2026-02-19T21:12:57.735 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99400.log 2026-02-19T21:12:57.736 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46054.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46054.log.gz 2026-02-19T21:12:57.736 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36743.log 2026-02-19T21:12:57.737 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99400.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99400.log.gz 2026-02-19T21:12:57.737 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111015.log 2026-02-19T21:12:57.738 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36743.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36743.log.gz 2026-02-19T21:12:57.738 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61263.log 2026-02-19T21:12:57.740 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.111015.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.111015.log.gz 2026-02-19T21:12:57.740 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22007.log 2026-02-19T21:12:57.741 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61263.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61263.log.gz 2026-02-19T21:12:57.741 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97006.log 2026-02-19T21:12:57.742 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22007.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22007.log.gz 2026-02-19T21:12:57.742 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116933.log 2026-02-19T21:12:57.743 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97006.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.97006.log.gz 2026-02-19T21:12:57.744 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42752.log 2026-02-19T21:12:57.745 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116933.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116933.log.gz 2026-02-19T21:12:57.745 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58023.log 2026-02-19T21:12:57.745 INFO:teuthology.orchestra.run.trial151.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.2.82200.log.gz 2026-02-19T21:12:57.745 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42752.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30761.log 2026-02-19T21:12:57.746 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58023.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42752.log.gz 2026-02-19T21:12:57.747 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58023.log.gz 2026-02-19T21:12:57.747 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26154.log 2026-02-19T21:12:57.747 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30761.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123691.log 2026-02-19T21:12:57.747 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30761.log.gz 2026-02-19T21:12:57.747 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85697.log 2026-02-19T21:12:57.751 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26154.log: /var/log/ceph/cluster1-client.mirror.123691.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26154.log.gz 2026-02-19T21:12:57.751 INFO:teuthology.orchestra.run.trial151.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123691.log.gz 2026-02-19T21:12:57.751 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62136.log 2026-02-19T21:12:57.751 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44989.log.gz 2026-02-19T21:12:57.751 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85697.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63362.log 2026-02-19T21:12:57.751 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85697.log.gz 2026-02-19T21:12:57.751 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62136.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62136.log.gz 2026-02-19T21:12:57.751 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88237.log 2026-02-19T21:12:57.751 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63362.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40166.log 2026-02-19T21:12:57.751 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63362.log.gz 2026-02-19T21:12:57.752 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59963.log 2026-02-19T21:12:57.752 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88237.log: /var/log/ceph/cluster2-client.mirror.40166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78779.log 2026-02-19T21:12:57.752 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88237.log.gz 2026-02-19T21:12:57.752 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59963.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40166.log.gz 2026-02-19T21:12:57.752 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82334.log 2026-02-19T21:12:57.752 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59963.log.gz 2026-02-19T21:12:57.752 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73390.log 2026-02-19T21:12:57.752 INFO:teuthology.orchestra.run.trial151.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78779.log.gz 2026-02-19T21:12:57.756 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.82334.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22379.log 2026-02-19T21:12:57.756 INFO:teuthology.orchestra.run.trial151.stderr: 84.8%/var/log/ceph/cluster2-client.mirror.73390.log: -- replaced with /var/log/ceph/cluster2-client.mirror.82334.log.gz 2026-02-19T21:12:57.756 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122367.log 2026-02-19T21:12:57.756 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73390.log.gz 2026-02-19T21:12:57.756 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22379.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45278.log 2026-02-19T21:12:57.756 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30856.log 2026-02-19T21:12:57.756 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22379.log.gz 2026-02-19T21:12:57.756 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45278.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45278.log.gz 2026-02-19T21:12:57.756 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121008.log 2026-02-19T21:12:57.757 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41039.log 2026-02-19T21:12:57.757 INFO:teuthology.orchestra.run.trial151.stderr: 90.6%/var/log/ceph/cluster1-client.mirror.30856.log: -- replaced with /var/log/ceph/cluster2-client.mirror.122367.log.gz 2026-02-19T21:12:57.757 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121008.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40651.log 2026-02-19T21:12:57.757 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41039.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41039.log.gz 2026-02-19T21:12:57.757 INFO:teuthology.orchestra.run.trial151.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121008.log.gz 2026-02-19T21:12:57.757 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48473.log 2026-02-19T21:12:57.757 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40651.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40651.log.gz 2026-02-19T21:12:57.758 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26480.log 2026-02-19T21:12:57.759 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48473.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80567.log 2026-02-19T21:12:57.759 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26480.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48473.log.gz 2026-02-19T21:12:57.759 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127646.log 2026-02-19T21:12:57.759 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26480.log.gz 2026-02-19T21:12:57.759 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.80567.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.30856.log.gz 2026-02-19T21:12:57.759 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.97704.log.gz 2026-02-19T21:12:57.773 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19941.log 2026-02-19T21:12:57.775 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117540.log 2026-02-19T21:12:57.775 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127646.log: /var/log/ceph/cluster2-client.admin.19941.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36584.log 2026-02-19T21:12:57.775 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19941.log.gz 2026-02-19T21:12:57.775 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.127646.log.gz 2026-02-19T21:12:57.775 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117540.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111488.log 2026-02-19T21:12:57.775 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51443.log 2026-02-19T21:12:57.777 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36584.log: /var/log/ceph/cluster2-client.mirror.111488.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111488.log.gz 2026-02-19T21:12:57.777 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36584.log.gz 2026-02-19T21:12:57.777 INFO:teuthology.orchestra.run.trial151.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.117540.log.gz 2026-02-19T21:12:57.777 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44242.log 2026-02-19T21:12:57.777 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123781.log 2026-02-19T21:12:57.777 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51443.log: /var/log/ceph/cluster2-client.mirror.44242.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51443.log.gz 2026-02-19T21:12:57.777 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44242.log.gz 2026-02-19T21:12:57.779 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58737.log 2026-02-19T21:12:57.779 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123781.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121944.log 2026-02-19T21:12:57.779 INFO:teuthology.orchestra.run.trial151.stderr: 87.1%/var/log/ceph/cluster2-client.mirror.58737.log: -- replaced with /var/log/ceph/cluster1-client.mirror.123781.log.gz 2026-02-19T21:12:57.779 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45957.log 2026-02-19T21:12:57.780 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58737.log.gz 2026-02-19T21:12:57.780 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121944.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.32805.log 2026-02-19T21:12:57.780 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.45957.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60157.log 2026-02-19T21:12:57.780 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.32805.log: 88.8% 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.121944.log.gz 2026-02-19T21:12:57.780 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.45957.log.gz 2026-02-19T21:12:57.780 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.80473.log 2026-02-19T21:12:57.781 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114231.log 2026-02-19T21:12:57.781 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.1.80473.log: /var/log/ceph/cluster2-client.mirror.60157.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60157.log.gz 2026-02-19T21:12:57.781 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128684.log 2026-02-19T21:12:57.782 INFO:teuthology.orchestra.run.trial151.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.0.32805.log.gz 2026-02-19T21:12:57.782 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114231.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114231.log.gz 2026-02-19T21:12:57.782 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42265.log 2026-02-19T21:12:57.783 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128684.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128684.log.gz 2026-02-19T21:12:57.783 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104612.log 2026-02-19T21:12:57.783 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42265.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55404.log 2026-02-19T21:12:57.784 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.104612.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.42265.log.gz 2026-02-19T21:12:57.784 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.104612.log.gz 2026-02-19T21:12:57.784 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118961.log 2026-02-19T21:12:57.784 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55404.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55404.log.gz 2026-02-19T21:12:57.784 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101398.log 2026-02-19T21:12:57.785 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118961.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118961.log.gz 2026-02-19T21:12:57.785 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70309.log 2026-02-19T21:12:57.785 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101398.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101398.log.gz 2026-02-19T21:12:57.786 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104545.log 2026-02-19T21:12:57.786 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70309.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.70309.log.gz 2026-02-19T21:12:57.787 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55986.log 2026-02-19T21:12:57.787 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104545.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104545.log.gz 2026-02-19T21:12:57.787 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44339.log 2026-02-19T21:12:57.788 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55986.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55986.log.gz 2026-02-19T21:12:57.788 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.19443.log 2026-02-19T21:12:57.789 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44339.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25766.log 2026-02-19T21:12:57.789 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.1.19443.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44339.log.gz 2026-02-19T21:12:57.789 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34512.log 2026-02-19T21:12:57.790 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25766.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.25766.log.gz 2026-02-19T21:12:57.790 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30445.log 2026-02-19T21:12:57.791 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34512.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34512.log.gz 2026-02-19T21:12:57.791 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28323.log 2026-02-19T21:12:57.792 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30445.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30445.log.gz 2026-02-19T21:12:57.792 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29588.log 2026-02-19T21:12:57.793 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28323.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28323.log.gz 2026-02-19T21:12:57.793 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124253.log 2026-02-19T21:12:57.793 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29588.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.19553.log 2026-02-19T21:12:57.793 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29588.log.gz 2026-02-19T21:12:57.794 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124253.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124253.log.gz 2026-02-19T21:12:57.794 INFO:teuthology.orchestra.run.trial151.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.1.80473.log.gz 2026-02-19T21:12:57.794 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125779.log 2026-02-19T21:12:57.795 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.1.19553.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51060.log 2026-02-19T21:12:57.795 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62392.log 2026-02-19T21:12:57.796 INFO:teuthology.orchestra.run.trial151.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125779.log.gz 2026-02-19T21:12:57.796 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20095.log 2026-02-19T21:12:57.797 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62392.log: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.1.19553.log.gz 2026-02-19T21:12:57.797 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62392.log.gz 2026-02-19T21:12:57.798 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.51060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40713.log 2026-02-19T21:12:57.798 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20095.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.20095.log.gz 2026-02-19T21:12:57.798 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33965.log 2026-02-19T21:12:57.799 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40713.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40713.log.gz 2026-02-19T21:12:57.799 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84623.log 2026-02-19T21:12:57.800 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33965.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33965.log.gz 2026-02-19T21:12:57.800 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55016.log 2026-02-19T21:12:57.800 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.84623.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84623.log.gz 2026-02-19T21:12:57.801 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose 93.6% -- /var/log/ceph/cluster2-client.mirror.57635.log -- replaced with /var/log/ceph/cluster1-client.mirror.51060.log.gz 2026-02-19T21:12:57.801 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:57.801 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55016.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55016.log.gz 2026-02-19T21:12:57.802 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84294.log 2026-02-19T21:12:57.802 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57635.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57635.log.gz 2026-02-19T21:12:57.802 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125318.log 2026-02-19T21:12:57.803 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.84294.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.84294.log.gz 2026-02-19T21:12:57.803 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44207.log 2026-02-19T21:12:57.805 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60584.log 2026-02-19T21:12:57.806 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44207.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44207.log.gz 2026-02-19T21:12:57.806 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96216.log 2026-02-19T21:12:57.807 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60584.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60584.log.gz 2026-02-19T21:12:57.807 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124208.log 2026-02-19T21:12:57.808 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96216.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96216.log.gz 2026-02-19T21:12:57.808 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123623.log 2026-02-19T21:12:57.809 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124208.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124208.log.gz 2026-02-19T21:12:57.809 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76491.log 2026-02-19T21:12:57.809 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123623.log: 85.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123623.log.gz 2026-02-19T21:12:57.810 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119968.log 2026-02-19T21:12:57.813 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76491.log: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.76491.log.gz 2026-02-19T21:12:57.813 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54275.log 2026-02-19T21:12:57.815 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119968.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119968.log.gz 2026-02-19T21:12:57.815 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69442.log 2026-02-19T21:12:57.816 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54275.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54275.log.gz 2026-02-19T21:12:57.816 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71023.log 2026-02-19T21:12:57.817 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69442.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69442.log.gz 2026-02-19T21:12:57.817 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61325.log 2026-02-19T21:12:57.817 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71023.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71023.log.gz 2026-02-19T21:12:57.818 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57379.log 2026-02-19T21:12:57.818 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61325.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61325.log.gz 2026-02-19T21:12:57.818 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20146.log 2026-02-19T21:12:57.819 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57379.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57379.log.gz 2026-02-19T21:12:57.819 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72676.log 2026-02-19T21:12:57.820 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20146.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101226.log 2026-02-19T21:12:57.820 INFO:teuthology.orchestra.run.trial151.stderr: 69.1% -- replaced with /var/log/ceph/cluster2-client.mirror.20146.log.gz 2026-02-19T21:12:57.821 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72676.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72676.log.gz 2026-02-19T21:12:57.821 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33445.log 2026-02-19T21:12:57.829 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101226.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19771.log 2026-02-19T21:12:57.841 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33445.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33445.log.gz 2026-02-19T21:12:57.845 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58349.log 2026-02-19T21:12:57.846 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.19771.log: 73.2% -- replaced with /var/log/ceph/cluster1-client.admin.19771.log.gz 2026-02-19T21:12:57.857 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69962.log.gz 2026-02-19T21:12:57.858 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73813.log 2026-02-19T21:12:57.859 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58349.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58349.log.gz 2026-02-19T21:12:57.859 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47988.log 2026-02-19T21:12:57.860 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73813.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73813.log.gz 2026-02-19T21:12:57.860 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36840.log 2026-02-19T21:12:57.861 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47988.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47988.log.gz 2026-02-19T21:12:57.861 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98792.log 2026-02-19T21:12:57.862 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36840.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36840.log.gz 2026-02-19T21:12:57.862 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86425.log 2026-02-19T21:12:57.863 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98792.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.98792.log.gz 2026-02-19T21:12:57.864 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71217.log 2026-02-19T21:12:57.864 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86425.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86425.log.gz 2026-02-19T21:12:57.865 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86015.log 2026-02-19T21:12:57.865 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71217.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71217.log.gz 2026-02-19T21:12:57.865 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104491.log 2026-02-19T21:12:57.866 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.86015.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86015.log.gz 2026-02-19T21:12:57.867 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119449.log 2026-02-19T21:12:57.868 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104491.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104491.log.gz 2026-02-19T21:12:57.868 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77077.log 2026-02-19T21:12:57.869 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119449.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119449.log.gz 2026-02-19T21:12:57.869 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49350.log 2026-02-19T21:12:57.873 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77077.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77077.log.gz 2026-02-19T21:12:57.874 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30526.log 2026-02-19T21:12:57.875 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49350.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49350.log.gz 2026-02-19T21:12:57.875 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80337.log.gz 2026-02-19T21:12:57.875 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44145.log 2026-02-19T21:12:57.875 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30526.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92718.log 2026-02-19T21:12:57.876 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44145.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.30526.log.gz 2026-02-19T21:12:57.876 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58252.log 2026-02-19T21:12:57.876 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44145.log.gz/var/log/ceph/cluster2-client.mirror.92718.log: 2026-02-19T21:12:57.877 INFO:teuthology.orchestra.run.trial151.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.92718.log.gz 2026-02-19T21:12:57.877 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18793.log 2026-02-19T21:12:57.877 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58252.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20129.log 2026-02-19T21:12:57.879 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58252.log.gz 2026-02-19T21:12:57.879 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18793.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18793.log.gz 2026-02-19T21:12:57.879 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80117.log 2026-02-19T21:12:57.879 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20129.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92076.log 2026-02-19T21:12:57.880 INFO:teuthology.orchestra.run.trial151.stderr: 64.8%/var/log/ceph/cluster2-client.mirror.80117.log: -- replaced with /var/log/ceph/cluster2-client.mirror.20129.log.gz 2026-02-19T21:12:57.880 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46345.log 2026-02-19T21:12:57.881 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92076.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.92076.log.gz 2026-02-19T21:12:57.881 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105437.log 2026-02-19T21:12:57.881 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46345.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46345.log.gz 2026-02-19T21:12:57.881 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20061.log 2026-02-19T21:12:57.882 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105437.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105437.log.gz 2026-02-19T21:12:57.883 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112144.log 2026-02-19T21:12:57.883 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20061.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.20061.log.gz 2026-02-19T21:12:57.883 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52479.log 2026-02-19T21:12:57.884 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112144.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112144.log.gz 2026-02-19T21:12:57.884 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61554.log 2026-02-19T21:12:57.885 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52479.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52479.log.gz 2026-02-19T21:12:57.885 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27062.log 2026-02-19T21:12:57.885 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61554.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61554.log.gz 2026-02-19T21:12:57.886 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123173.log 2026-02-19T21:12:57.887 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27062.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27062.log.gz 2026-02-19T21:12:57.887 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.19587.log 2026-02-19T21:12:57.887 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123173.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123173.log.gz 2026-02-19T21:12:57.887 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29491.log 2026-02-19T21:12:57.888 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.3.19587.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97297.log 2026-02-19T21:12:57.897 INFO:teuthology.orchestra.run.trial151.stderr: 93.5%/var/log/ceph/cluster2-client.mirror.29491.log: -- replaced with /var/log/ceph/cluster2-client.mirror.3.19587.log.gz 2026-02-19T21:12:57.897 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29491.log.gz 2026-02-19T21:12:57.897 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97727.log 2026-02-19T21:12:57.898 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97297.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97297.log.gz 2026-02-19T21:12:57.898 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59575.log 2026-02-19T21:12:57.901 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.97727.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30944.log.gz 2026-02-19T21:12:57.901 INFO:teuthology.orchestra.run.trial151.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.mirror.97727.log.gz 2026-02-19T21:12:57.901 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39840.log 2026-02-19T21:12:57.902 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59575.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95934.log 2026-02-19T21:12:57.902 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59575.log.gz 2026-02-19T21:12:57.903 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39840.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109697.log 2026-02-19T21:12:57.903 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95934.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39840.log.gz 2026-02-19T21:12:57.904 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121380.log 2026-02-19T21:12:57.904 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109697.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.95934.log.gz 2026-02-19T21:12:57.904 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109697.log.gz 2026-02-19T21:12:57.904 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75649.log 2026-02-19T21:12:57.905 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121380.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121380.log.gz 2026-02-19T21:12:57.905 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40907.log 2026-02-19T21:12:57.905 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75649.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89670.log 2026-02-19T21:12:57.906 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40907.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40907.log.gz 2026-02-19T21:12:57.914 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.108740.log.gz 2026-02-19T21:12:57.914 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109870.log 2026-02-19T21:12:57.914 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89670.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89670.log.gz 2026-02-19T21:12:57.915 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89635.log 2026-02-19T21:12:57.916 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109870.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.109870.log.gz 2026-02-19T21:12:57.916 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95632.log 2026-02-19T21:12:57.916 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89635.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89635.log.gz 2026-02-19T21:12:57.917 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120455.log 2026-02-19T21:12:57.917 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95632.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22716.log 2026-02-19T21:12:57.917 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120455.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95632.log.gz 2026-02-19T21:12:57.918 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116441.log 2026-02-19T21:12:57.918 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120455.log.gz 2026-02-19T21:12:57.918 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22716.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30364.log 2026-02-19T21:12:57.918 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116441.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22716.log.gz 2026-02-19T21:12:57.919 INFO:teuthology.orchestra.run.trial151.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116441.log.gz 2026-02-19T21:12:57.919 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60840.log 2026-02-19T21:12:57.919 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30364.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76359.log 2026-02-19T21:12:57.920 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60840.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60840.log.gz 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30364.log.gz 2026-02-19T21:12:57.920 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:57.920 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61810.log 2026-02-19T21:12:57.920 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76359.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124343.log 2026-02-19T21:12:57.920 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61810.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76359.log.gz 2026-02-19T21:12:57.922 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61810.log.gz 2026-02-19T21:12:57.922 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98413.log 2026-02-19T21:12:57.922 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124343.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124343.log.gz 2026-02-19T21:12:57.922 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21728.log 2026-02-19T21:12:57.924 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98413.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105768.log 2026-02-19T21:12:57.924 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21728.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98413.log.gz 2026-02-19T21:12:57.924 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122723.log 2026-02-19T21:12:57.924 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21728.log.gz 2026-02-19T21:12:57.924 INFO:teuthology.orchestra.run.trial151.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75649.log.gz 2026-02-19T21:12:57.924 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105768.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94964.log 2026-02-19T21:12:57.924 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105768.log.gz 2026-02-19T21:12:57.924 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122723.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112009.log 2026-02-19T21:12:57.924 INFO:teuthology.orchestra.run.trial151.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.122723.log.gz 2026-02-19T21:12:57.925 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94964.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116712.log 2026-02-19T21:12:57.925 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40845.log 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112009.log: /var/log/ceph/cluster2-client.mirror.116712.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.112009.log.gz 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116712.log.gz 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27900.log 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.94964.log.gz 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100383.log 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112761.log 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27900.log: /var/log/ceph/cluster1-client.mirror.100383.log: gzip 88.7% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.40845.log.gz 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.120426.log 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112761.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27900.log.gz 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.100383.log.gz 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112761.log.gz 2026-02-19T21:12:57.927 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122442.log 2026-02-19T21:12:57.928 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118841.log 2026-02-19T21:12:57.928 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120426.log: /var/log/ceph/cluster2-client.mirror.122442.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.120426.log.gz 2026-02-19T21:12:57.928 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63106.log 2026-02-19T21:12:57.928 INFO:teuthology.orchestra.run.trial151.stderr: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.122442.log.gz 2026-02-19T21:12:57.929 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118841.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69692.log 2026-02-19T21:12:57.929 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63106.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118841.log.gz 2026-02-19T21:12:57.929 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122791.log 2026-02-19T21:12:57.930 INFO:teuthology.orchestra.run.trial151.stderr: 88.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35870.log 2026-02-19T21:12:57.930 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.63106.log.gz 2026-02-19T21:12:57.930 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69692.log: /var/log/ceph/cluster1-client.mirror.122791.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105814.log 2026-02-19T21:12:57.931 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35870.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.122791.log.gz 2026-02-19T21:12:57.931 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35870.log.gz 2026-02-19T21:12:57.931 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76471.log 2026-02-19T21:12:57.931 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105814.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125382.log 2026-02-19T21:12:57.931 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105814.log.gz 2026-02-19T21:12:57.932 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76471.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122926.log 2026-02-19T21:12:57.932 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125382.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125382.log.gz 2026-02-19T21:12:57.933 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119339.log 2026-02-19T21:12:57.933 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122926.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.122926.log.gz 2026-02-19T21:12:57.933 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97127.log 2026-02-19T21:12:57.934 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.119339.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.119339.log.gz 2026-02-19T21:12:57.934 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112676.log 2026-02-19T21:12:57.935 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97127.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.97127.log.gz 2026-02-19T21:12:57.935 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104057.log 2026-02-19T21:12:57.935 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112676.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112676.log.gz 2026-02-19T21:12:57.936 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19967.log 2026-02-19T21:12:57.936 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104057.log: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76471.log.gz 2026-02-19T21:12:57.937 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88329.log 2026-02-19T21:12:57.937 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19967.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19967.log.gz 2026-02-19T21:12:57.937 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126932.log 2026-02-19T21:12:57.939 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88329.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88329.log.gz 2026-02-19T21:12:57.939 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62718.log 2026-02-19T21:12:57.941 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126932.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.126932.log.gz 2026-02-19T21:12:57.941 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128316.log 2026-02-19T21:12:57.942 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62718.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62718.log.gz 2026-02-19T21:12:57.942 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27706.log 2026-02-19T21:12:57.943 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.128316.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.128316.log.gz 2026-02-19T21:12:57.943 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117515.log 2026-02-19T21:12:57.944 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27706.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.27706.log.gz 2026-02-19T21:12:57.944 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33868.log 2026-02-19T21:12:57.945 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.117515.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.117515.log.gz 2026-02-19T21:12:57.945 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55889.log 2026-02-19T21:12:57.946 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33868.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33868.log.gz 2026-02-19T21:12:57.946 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37902.log 2026-02-19T21:12:57.947 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55889.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55889.log.gz 2026-02-19T21:12:57.947 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112881.log 2026-02-19T21:12:57.948 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37902.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37902.log.gz 2026-02-19T21:12:57.948 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65610.log 2026-02-19T21:12:57.949 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112881.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112881.log.gz 2026-02-19T21:12:57.949 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119145.log 2026-02-19T21:12:57.950 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65610.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65610.log.gz 2026-02-19T21:12:57.950 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31934.log 2026-02-19T21:12:57.952 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119145.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119145.log.gz 2026-02-19T21:12:57.952 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30811.log 2026-02-19T21:12:57.953 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31934.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31934.log.gz 2026-02-19T21:12:57.953 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62815.log 2026-02-19T21:12:57.954 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.30811.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48535.log 2026-02-19T21:12:57.954 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.30811.log.gz 2026-02-19T21:12:57.955 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.62815.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62815.log.gz 2026-02-19T21:12:57.955 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72385.log 2026-02-19T21:12:57.956 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48535.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48535.log.gz 2026-02-19T21:12:57.956 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19924.log 2026-02-19T21:12:57.957 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72385.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72385.log.gz 2026-02-19T21:12:57.957 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91349.log 2026-02-19T21:12:57.957 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19924.log: 54.2% -- replaced with /var/log/ceph/cluster1-client.mirror.19924.log.gz 2026-02-19T21:12:57.958 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27159.log 2026-02-19T21:12:57.965 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.91349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114400.log 2026-02-19T21:12:57.982 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27159.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27159.log.gz 2026-02-19T21:12:57.982 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123331.log 2026-02-19T21:12:57.982 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114400.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114400.log.gz 2026-02-19T21:12:57.982 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35517.log 2026-02-19T21:12:57.983 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123331.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123331.log.gz 2026-02-19T21:12:57.989 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78400.log 2026-02-19T21:12:58.006 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35517.log: 88.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103030.log 2026-02-19T21:12:58.006 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.35517.log.gz 2026-02-19T21:12:58.006 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78400.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78400.log.gz 2026-02-19T21:12:58.011 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.101226.log.gz 2026-02-19T21:12:58.011 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69934.log 2026-02-19T21:12:58.012 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.103030.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.103030.log.gz 2026-02-19T21:12:58.012 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39580.log 2026-02-19T21:12:58.012 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.69934.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.69934.log.gz 2026-02-19T21:12:58.013 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77396.log 2026-02-19T21:12:58.014 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39580.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39580.log.gz 2026-02-19T21:12:58.014 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101887.log 2026-02-19T21:12:58.014 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77396.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77396.log.gz 2026-02-19T21:12:58.015 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39972.log 2026-02-19T21:12:58.016 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.101887.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.101887.log.gz 2026-02-19T21:12:58.017 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123421.log 2026-02-19T21:12:58.017 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39972.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39972.log.gz 2026-02-19T21:12:58.017 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100106.log 2026-02-19T21:12:58.018 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123421.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123421.log.gz 2026-02-19T21:12:58.018 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20010.log 2026-02-19T21:12:58.019 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100106.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100106.log.gz 2026-02-19T21:12:58.019 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127909.log 2026-02-19T21:12:58.019 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.20010.log: 80.4% -- replaced with /var/log/ceph/cluster2-client.admin.20010.log.gz 2026-02-19T21:12:58.020 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122836.log 2026-02-19T21:12:58.020 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127909.log: 83.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127909.log.gz 2026-02-19T21:12:58.020 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125210.log 2026-02-19T21:12:58.021 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122836.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122836.log.gz 2026-02-19T21:12:58.021 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110130.log 2026-02-19T21:12:58.022 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125210.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125210.log.gz 2026-02-19T21:12:58.022 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121059.log 2026-02-19T21:12:58.024 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.110130.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.110130.log.gz 2026-02-19T21:12:58.025 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43563.log 2026-02-19T21:12:58.025 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121059.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.121059.log.gz 2026-02-19T21:12:58.025 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54954.log 2026-02-19T21:12:58.026 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43563.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43563.log.gz 2026-02-19T21:12:58.026 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34803.log 2026-02-19T21:12:58.027 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54954.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54954.log.gz 2026-02-19T21:12:58.027 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123488.log 2026-02-19T21:12:58.028 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34803.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34803.log.gz 2026-02-19T21:12:58.028 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115111.log 2026-02-19T21:12:58.028 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123488.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123488.log.gz 2026-02-19T21:12:58.029 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106514.log 2026-02-19T21:12:58.029 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115111.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115111.log.gz 2026-02-19T21:12:58.029 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105155.log 2026-02-19T21:12:58.030 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.106514.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.106514.log.gz 2026-02-19T21:12:58.030 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68908.log 2026-02-19T21:12:58.031 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105155.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105155.log.gz 2026-02-19T21:12:58.031 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65139.log 2026-02-19T21:12:58.031 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68908.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68908.log.gz 2026-02-19T21:12:58.032 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117103.log 2026-02-19T21:12:58.033 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.65139.log: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65139.log.gz 2026-02-19T21:12:58.033 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89256.log 2026-02-19T21:12:58.033 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117103.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.117103.log.gz 2026-02-19T21:12:58.034 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101909.log 2026-02-19T21:12:58.035 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89256.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89256.log.gz 2026-02-19T21:12:58.035 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121238.log 2026-02-19T21:12:58.050 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101909.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82998.log.gz 2026-02-19T21:12:58.050 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127694.log.gz 2026-02-19T21:12:58.051 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38963.log 2026-02-19T21:12:58.051 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121238.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25281.log 2026-02-19T21:12:58.051 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38963.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.121238.log.gz 2026-02-19T21:12:58.051 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38963.log.gz 2026-02-19T21:12:58.053 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127237.log 2026-02-19T21:12:58.053 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25281.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128391.log 2026-02-19T21:12:58.054 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127237.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25281.log.gz 2026-02-19T21:12:58.054 INFO:teuthology.orchestra.run.trial151.stderr: 83.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127237.log.gz 2026-02-19T21:12:58.054 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124231.log 2026-02-19T21:12:58.054 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.128391.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125189.log 2026-02-19T21:12:58.054 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.128391.log.gz 2026-02-19T21:12:58.054 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124231.log: gzip 87.1% -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51831.log 2026-02-19T21:12:58.054 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.124231.log.gz 2026-02-19T21:12:58.054 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125189.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125189.log.gz 2026-02-19T21:12:58.054 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122701.log 2026-02-19T21:12:58.054 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37747.log 2026-02-19T21:12:58.055 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122701.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51831.log.gz 2026-02-19T21:12:58.056 INFO:teuthology.orchestra.run.trial151.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122701.log.gz 2026-02-19T21:12:58.056 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78098.log 2026-02-19T21:12:58.056 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37747.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93557.log 2026-02-19T21:12:58.056 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78098.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106800.log 2026-02-19T21:12:58.065 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37166.log 2026-02-19T21:12:58.074 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93557.log: /var/log/ceph/cluster2-client.mirror.106800.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93557.log.gz 2026-02-19T21:12:58.077 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118719.log 2026-02-19T21:12:58.089 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.37166.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37166.log.gz 2026-02-19T21:12:58.093 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41815.log 2026-02-19T21:12:58.095 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118719.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.118719.log.gz 2026-02-19T21:12:58.109 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126442.log 2026-02-19T21:12:58.121 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41815.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41815.log.gz 2026-02-19T21:12:58.124 INFO:teuthology.orchestra.run.trial151.stderr: 95.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.19427.log.gz 2026-02-19T21:12:58.125 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28129.log 2026-02-19T21:12:58.126 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126442.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126442.log.gz 2026-02-19T21:12:58.134 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29068.log 2026-02-19T21:12:58.146 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28129.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28129.log.gz 2026-02-19T21:12:58.149 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109440.log 2026-02-19T21:12:58.150 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29068.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29068.log.gz 2026-02-19T21:12:58.166 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75074.log 2026-02-19T21:12:58.167 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109440.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.109440.log.gz 2026-02-19T21:12:58.179 INFO:teuthology.orchestra.run.trial151.stderr: 95.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111642.log 2026-02-19T21:12:58.179 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75074.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75074.log.gz 2026-02-19T21:12:58.179 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.106800.log.gz 2026-02-19T21:12:58.179 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33348.log 2026-02-19T21:12:58.180 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.111642.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111642.log.gz 2026-02-19T21:12:58.180 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36452.log 2026-02-19T21:12:58.184 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.33348.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23351.log 2026-02-19T21:12:58.185 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33348.log.gz 2026-02-19T21:12:58.185 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36452.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36452.log.gz 2026-02-19T21:12:58.185 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120937.log 2026-02-19T21:12:58.185 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23351.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66739.log 2026-02-19T21:12:58.185 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23351.log.gz 2026-02-19T21:12:58.185 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120937.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.120937.log.gz 2026-02-19T21:12:58.185 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74845.log 2026-02-19T21:12:58.185 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66739.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66739.log.gz 2026-02-19T21:12:58.186 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27547.log 2026-02-19T21:12:58.195 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74845.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91349.log.gz 2026-02-19T21:12:58.195 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74845.log.gz 2026-02-19T21:12:58.195 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57088.log 2026-02-19T21:12:58.196 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.27547.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27838.log 2026-02-19T21:12:58.196 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57088.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27547.log.gz 2026-02-19T21:12:58.196 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57088.log.gz 2026-02-19T21:12:58.196 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94535.log 2026-02-19T21:12:58.196 INFO:teuthology.orchestra.run.trial151.stderr: 95.7%/var/log/ceph/cluster2-client.mirror.27838.log: -- replaced with /var/log/ceph/cluster2-client.mirror.37747.log.gz 2026-02-19T21:12:58.196 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128606.log 2026-02-19T21:12:58.196 INFO:teuthology.orchestra.run.trial151.stderr: 88.4%/var/log/ceph/cluster2-client.mirror.94535.log: -- replaced with /var/log/ceph/cluster2-client.mirror.27838.log.gz 2026-02-19T21:12:58.196 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94535.log.gz 2026-02-19T21:12:58.196 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125543.log 2026-02-19T21:12:58.196 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24408.log 2026-02-19T21:12:58.197 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128606.log: /var/log/ceph/cluster2-client.mirror.125543.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128606.log.gz 2026-02-19T21:12:58.197 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66933.log 2026-02-19T21:12:58.197 INFO:teuthology.orchestra.run.trial151.stderr: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125543.log.gz 2026-02-19T21:12:58.197 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58702.log 2026-02-19T21:12:58.197 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24408.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115289.log 2026-02-19T21:12:58.197 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66933.log: 89.0% 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24408.log.gz 2026-02-19T21:12:58.197 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58702.log: -- replaced with /var/log/ceph/cluster2-client.mirror.66933.log.gz 2026-02-19T21:12:58.197 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58702.log.gz 2026-02-19T21:12:58.197 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95414.log 2026-02-19T21:12:58.197 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115289.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40131.log 2026-02-19T21:12:58.198 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95414.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.115289.log.gz 2026-02-19T21:12:58.198 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95414.log.gz 2026-02-19T21:12:58.198 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15970.log 2026-02-19T21:12:58.198 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40131.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76405.log 2026-02-19T21:12:58.198 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.15970.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85329.log 2026-02-19T21:12:58.198 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15970.log.gz 2026-02-19T21:12:58.198 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40131.log.gz 2026-02-19T21:12:58.198 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76405.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119003.log 2026-02-19T21:12:58.198 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121037.log 2026-02-19T21:12:58.200 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119003.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.119003.log.gz 2026-02-19T21:12:58.200 INFO:teuthology.orchestra.run.trial151.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76405.log.gz 2026-02-19T21:12:58.200 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56665.log 2026-02-19T21:12:58.200 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57185.log 2026-02-19T21:12:58.200 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121037.log: /var/log/ceph/cluster2-client.mirror.56665.log: 89.4% 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56665.log.gz 2026-02-19T21:12:58.200 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.121037.log.gz 2026-02-19T21:12:58.209 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102513.log 2026-02-19T21:12:58.210 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57185.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77829.log 2026-02-19T21:12:58.210 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57185.log.gz 2026-02-19T21:12:58.210 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102513.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112373.log 2026-02-19T21:12:58.211 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77829.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.102513.log.gz 2026-02-19T21:12:58.211 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105536.log 2026-02-19T21:12:58.213 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112373.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.112373.log.gz 2026-02-19T21:12:58.213 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104015.log 2026-02-19T21:12:58.229 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105536.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91250.log 2026-02-19T21:12:58.231 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104015.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.104015.log.gz 2026-02-19T21:12:58.237 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120988.log 2026-02-19T21:12:58.238 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.91250.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.91250.log.gz 2026-02-19T21:12:58.258 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116553.log 2026-02-19T21:12:58.258 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120988.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83575.log.gz 2026-02-19T21:12:58.258 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.120988.log.gz 2026-02-19T21:12:58.259 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39677.log 2026-02-19T21:12:58.259 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.116553.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.116553.log.gz 2026-02-19T21:12:58.259 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76839.log 2026-02-19T21:12:58.260 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39677.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39677.log.gz 2026-02-19T21:12:58.260 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125481.log 2026-02-19T21:12:58.263 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.76839.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.76839.log.gz 2026-02-19T21:12:58.264 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97528.log 2026-02-19T21:12:58.264 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125481.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125481.log.gz 2026-02-19T21:12:58.265 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122488.log 2026-02-19T21:12:58.265 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97528.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97528.log.gz 2026-02-19T21:12:58.265 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89921.log 2026-02-19T21:12:58.266 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122488.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122488.log.gz 2026-02-19T21:12:58.266 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92956.log 2026-02-19T21:12:58.267 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.89921.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.89921.log.gz 2026-02-19T21:12:58.267 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57926.log 2026-02-19T21:12:58.267 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.92956.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92956.log.gz 2026-02-19T21:12:58.268 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113412.log 2026-02-19T21:12:58.268 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57926.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57926.log.gz 2026-02-19T21:12:58.269 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59901.log 2026-02-19T21:12:58.269 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113412.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.113412.log.gz 2026-02-19T21:12:58.269 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54469.log 2026-02-19T21:12:58.270 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59901.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59901.log.gz 2026-02-19T21:12:58.270 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.19526.log 2026-02-19T21:12:58.271 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54469.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54469.log.gz 2026-02-19T21:12:58.271 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77417.log 2026-02-19T21:12:58.272 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.0.19526.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32613.log 2026-02-19T21:12:58.277 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.105536.log.gz 2026-02-19T21:12:58.278 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72946.log 2026-02-19T21:12:58.279 INFO:teuthology.orchestra.run.trial151.stderr: 92.3% 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82357.log.gz 2026-02-19T21:12:58.279 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32613.log: /var/log/ceph/cluster2-client.mirror.77417.log: -- replaced with /var/log/ceph/cluster2-client.mirror.0.19526.log.gz 2026-02-19T21:12:58.279 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32613.log.gz 2026-02-19T21:12:58.280 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39448.log 2026-02-19T21:12:58.280 INFO:teuthology.orchestra.run.trial151.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77417.log.gz 2026-02-19T21:12:58.280 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.104057.log.gz 2026-02-19T21:12:58.280 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72946.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43043.log 2026-02-19T21:12:58.280 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39448.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52673.log 2026-02-19T21:12:58.280 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43043.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39448.log.gz 2026-02-19T21:12:58.281 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22158.log 2026-02-19T21:12:58.281 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24734.log 2026-02-19T21:12:58.281 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22158.log: 88.7% 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52673.log.gz 2026-02-19T21:12:58.281 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.43043.log.gz 2026-02-19T21:12:58.282 INFO:teuthology.orchestra.run.trial151.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.22158.log.gz 2026-02-19T21:12:58.282 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76772.log 2026-02-19T21:12:58.282 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40422.log 2026-02-19T21:12:58.283 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76772.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24734.log.gz 2026-02-19T21:12:58.283 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85826.log 2026-02-19T21:12:58.283 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.40422.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43078.log 2026-02-19T21:12:58.284 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85826.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40422.log.gz 2026-02-19T21:12:58.284 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85826.log.gz 2026-02-19T21:12:58.284 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19276.log 2026-02-19T21:12:58.293 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43078.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21337.log 2026-02-19T21:12:58.294 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.19276.log: 0.0% 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43078.log.gz 2026-02-19T21:12:58.294 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.19276.log.gz 2026-02-19T21:12:58.294 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21914.log 2026-02-19T21:12:58.294 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.21337.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24443.log 2026-02-19T21:12:58.294 INFO:teuthology.orchestra.run.trial151.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.76772.log.gz 2026-02-19T21:12:58.294 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21914.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.21337.log.gz 2026-02-19T21:12:58.294 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.21914.log.gz 2026-02-19T21:12:58.295 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30899.log 2026-02-19T21:12:58.295 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53708.log 2026-02-19T21:12:58.295 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.30899.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46601.log 2026-02-19T21:12:58.295 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24443.log.gz 2026-02-19T21:12:58.295 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53708.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53708.log.gz 2026-02-19T21:12:58.295 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122768.log 2026-02-19T21:12:58.295 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46601.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46601.log.gz 2026-02-19T21:12:58.295 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39289.log 2026-02-19T21:12:58.295 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.122768.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90604.log 2026-02-19T21:12:58.296 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.122768.log.gz 2026-02-19T21:12:58.296 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.39289.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121538.log 2026-02-19T21:12:58.296 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90604.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39289.log.gz 2026-02-19T21:12:58.296 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.90604.log.gz 2026-02-19T21:12:58.296 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81865.log 2026-02-19T21:12:58.296 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121538.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67127.log 2026-02-19T21:12:58.296 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81865.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121538.log.gz 2026-02-19T21:12:58.296 INFO:teuthology.orchestra.run.trial151.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.30899.log.gz 2026-02-19T21:12:58.297 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41912.log 2026-02-19T21:12:58.297 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67127.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60778.log 2026-02-19T21:12:58.297 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41912.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67127.log.gz 2026-02-19T21:12:58.297 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78075.log 2026-02-19T21:12:58.297 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41912.log.gz 2026-02-19T21:12:58.297 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.60778.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60778.log.gz 2026-02-19T21:12:58.297 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51125.log 2026-02-19T21:12:58.297 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78075.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78075.log.gz 2026-02-19T21:12:58.299 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54070.log 2026-02-19T21:12:58.300 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51125.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124930.log 2026-02-19T21:12:58.300 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54070.log: 96.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51125.log.gz 2026-02-19T21:12:58.300 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102203.log 2026-02-19T21:12:58.300 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124930.log: 63.2% -- replaced with /var/log/ceph/cluster1-client.mirror.124930.log.gz 2026-02-19T21:12:58.300 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122325.log 2026-02-19T21:12:58.300 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102203.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102203.log.gz 2026-02-19T21:12:58.301 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49994.log 2026-02-19T21:12:58.302 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122325.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.122325.log.gz 2026-02-19T21:12:58.302 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56180.log 2026-02-19T21:12:58.302 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.49994.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49994.log.gz 2026-02-19T21:12:58.303 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118819.log 2026-02-19T21:12:58.303 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56180.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56180.log.gz 2026-02-19T21:12:58.303 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90158.log 2026-02-19T21:12:58.303 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118819.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118819.log.gz 2026-02-19T21:12:58.304 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41004.log 2026-02-19T21:12:58.304 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.90158.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.90158.log.gz 2026-02-19T21:12:58.305 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78357.log 2026-02-19T21:12:58.306 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.41004.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41004.log.gz 2026-02-19T21:12:58.306 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100025.log 2026-02-19T21:12:58.319 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78357.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37659.log.gz 2026-02-19T21:12:58.319 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119926.log 2026-02-19T21:12:58.320 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100025.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100025.log.gz 2026-02-19T21:12:58.320 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19754.log 2026-02-19T21:12:58.321 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119926.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119926.log.gz 2026-02-19T21:12:58.321 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.97727.log 2026-02-19T21:12:58.322 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.19754.log: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.19754.log.gz 2026-02-19T21:12:58.322 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38478.log 2026-02-19T21:12:58.338 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.1.97727.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108698.log 2026-02-19T21:12:58.338 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38478.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38478.log.gz 2026-02-19T21:12:58.346 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77589.log 2026-02-19T21:12:58.346 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108698.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.108698.log.gz 2026-02-19T21:12:58.347 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84317.log.gz 2026-02-19T21:12:58.348 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100222.log 2026-02-19T21:12:58.348 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77589.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77589.log.gz 2026-02-19T21:12:58.349 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127521.log 2026-02-19T21:12:58.353 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100222.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100222.log.gz 2026-02-19T21:12:58.353 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68000.log 2026-02-19T21:12:58.353 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.127521.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.127521.log.gz 2026-02-19T21:12:58.353 INFO:teuthology.orchestra.run.trial151.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.1.97727.log.gz 2026-02-19T21:12:58.354 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47662.log 2026-02-19T21:12:58.354 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68000.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99549.log 2026-02-19T21:12:58.354 INFO:teuthology.orchestra.run.trial151.stderr: 88.7%/var/log/ceph/cluster2-client.mirror.47662.log: -- replaced with /var/log/ceph/cluster2-client.mirror.68000.log.gz 2026-02-19T21:12:58.354 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47662.log.gz 2026-02-19T21:12:58.354 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76796.log 2026-02-19T21:12:58.354 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22530.log 2026-02-19T21:12:58.354 INFO:teuthology.orchestra.run.trial151.stderr: 88.2%/var/log/ceph/cluster1-client.mirror.76796.log: -- replaced with /var/log/ceph/cluster2-client.mirror.99549.log.gz 2026-02-19T21:12:58.354 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128341.log 2026-02-19T21:12:58.355 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22530.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22530.log.gz 2026-02-19T21:12:58.356 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51083.log 2026-02-19T21:12:58.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.128341.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.128341.log.gz 2026-02-19T21:12:58.356 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34318.log 2026-02-19T21:12:58.356 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51083.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.76796.log.gz 2026-02-19T21:12:58.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86283.log 2026-02-19T21:12:58.357 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34318.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34318.log.gz 2026-02-19T21:12:58.357 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66192.log 2026-02-19T21:12:58.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.86283.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86283.log.gz 2026-02-19T21:12:58.359 INFO:teuthology.orchestra.run.trial151.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51083.log.gz 2026-02-19T21:12:58.359 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69067.log 2026-02-19T21:12:58.359 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66192.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20112.log 2026-02-19T21:12:58.359 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66192.log.gz/var/log/ceph/cluster2-client.mirror.69067.log: 2026-02-19T21:12:58.359 INFO:teuthology.orchestra.run.trial151.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69067.log.gz 2026-02-19T21:12:58.359 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119316.log 2026-02-19T21:12:58.360 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20112.log: 68.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98347.log 2026-02-19T21:12:58.361 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.20112.log.gz 2026-02-19T21:12:58.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29685.log 2026-02-19T21:12:58.361 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.98347.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.119316.log.gz 2026-02-19T21:12:58.361 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31899.log 2026-02-19T21:12:58.362 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.29685.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29685.log.gz 2026-02-19T21:12:58.362 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120639.log 2026-02-19T21:12:58.362 INFO:teuthology.orchestra.run.trial151.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.98347.log.gz 2026-02-19T21:12:58.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31899.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31899.log.gz 2026-02-19T21:12:58.363 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56471.log 2026-02-19T21:12:58.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.120639.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118322.log 2026-02-19T21:12:58.363 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56471.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56471.log.gz 2026-02-19T21:12:58.364 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125006.log 2026-02-19T21:12:58.364 INFO:teuthology.orchestra.run.trial151.stderr: 90.7%/var/log/ceph/cluster1-client.mirror.118322.log: -- replaced with /var/log/ceph/cluster2-client.mirror.120639.log.gz 2026-02-19T21:12:58.365 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.118322.log.gz 2026-02-19T21:12:58.365 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119591.log 2026-02-19T21:12:58.365 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32190.log 2026-02-19T21:12:58.365 INFO:teuthology.orchestra.run.trial151.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125006.log.gz 2026-02-19T21:12:58.366 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119591.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81082.log 2026-02-19T21:12:58.366 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32190.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126289.log 2026-02-19T21:12:58.366 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32190.log.gz 2026-02-19T21:12:58.366 INFO:teuthology.orchestra.run.trial151.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.119591.log.gz 2026-02-19T21:12:58.366 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.81082.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109569.log 2026-02-19T21:12:58.367 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126289.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126289.log.gz 2026-02-19T21:12:58.367 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115459.log 2026-02-19T21:12:58.368 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109569.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109569.log.gz 2026-02-19T21:12:58.368 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70732.log 2026-02-19T21:12:58.368 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.115459.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.115459.log.gz 2026-02-19T21:12:58.369 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63750.log 2026-02-19T21:12:58.369 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70732.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70732.log.gz 2026-02-19T21:12:58.370 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85970.log 2026-02-19T21:12:58.370 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63750.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63750.log.gz 2026-02-19T21:12:58.371 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19376.log 2026-02-19T21:12:58.371 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.85970.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85970.log.gz 2026-02-19T21:12:58.372 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93619.log 2026-02-19T21:12:58.372 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19376.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19376.log.gz 2026-02-19T21:12:58.372 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112094.log 2026-02-19T21:12:58.373 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93619.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93619.log.gz 2026-02-19T21:12:58.373 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43854.log 2026-02-19T21:12:58.374 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.112094.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.112094.log.gz 2026-02-19T21:12:58.374 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73231.log 2026-02-19T21:12:58.375 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.43854.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43854.log.gz 2026-02-19T21:12:58.375 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37614.log 2026-02-19T21:12:58.376 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73231.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73231.log.gz 2026-02-19T21:12:58.376 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23123.log 2026-02-19T21:12:58.377 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.37614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119811.log 2026-02-19T21:12:58.380 INFO:teuthology.orchestra.run.trial151.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37614.log.gz 2026-02-19T21:12:58.381 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.23123.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23123.log.gz 2026-02-19T21:12:58.381 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113489.log 2026-02-19T21:12:58.382 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119811.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.119811.log.gz 2026-02-19T21:12:58.382 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93041.log 2026-02-19T21:12:58.383 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113489.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113489.log.gz 2026-02-19T21:12:58.383 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128710.log 2026-02-19T21:12:58.384 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93041.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93041.log.gz 2026-02-19T21:12:58.384 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95713.log 2026-02-19T21:12:58.385 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128710.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128710.log.gz 2026-02-19T21:12:58.385 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51040.log 2026-02-19T21:12:58.386 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.95713.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95713.log.gz 2026-02-19T21:12:58.386 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72288.log 2026-02-19T21:12:58.390 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51040.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51040.log.gz 2026-02-19T21:12:58.390 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76557.log 2026-02-19T21:12:58.391 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72288.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72288.log.gz 2026-02-19T21:12:58.391 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.12214.log 2026-02-19T21:12:58.395 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76557.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76557.log.gz 2026-02-19T21:12:58.395 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124883.log 2026-02-19T21:12:58.395 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/ceph.tmp-client.admin.12214.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.12214.log.gz 2026-02-19T21:12:58.396 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122508.log 2026-02-19T21:12:58.396 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.124883.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.124883.log.gz 2026-02-19T21:12:58.396 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64429.log 2026-02-19T21:12:58.398 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.122508.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.122508.log.gz 2026-02-19T21:12:58.398 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106227.log 2026-02-19T21:12:58.399 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.64429.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64429.log.gz 2026-02-19T21:12:58.399 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20163.log 2026-02-19T21:12:58.400 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.106227.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.106227.log.gz 2026-02-19T21:12:58.401 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79939.log 2026-02-19T21:12:58.401 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20163.log: 53.8% -- replaced with /var/log/ceph/cluster2-client.mirror.20163.log.gz 2026-02-19T21:12:58.402 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38121.log 2026-02-19T21:12:58.415 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79939.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.101909.log.gz 2026-02-19T21:12:58.415 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44013.log 2026-02-19T21:12:58.416 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38121.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38121.log.gz 2026-02-19T21:12:58.416 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121802.log 2026-02-19T21:12:58.417 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44013.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44013.log.gz 2026-02-19T21:12:58.417 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47697.log 2026-02-19T21:12:58.418 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.121802.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.121802.log.gz 2026-02-19T21:12:58.419 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127347.log 2026-02-19T21:12:58.419 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47697.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47697.log.gz 2026-02-19T21:12:58.419 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75825.log 2026-02-19T21:12:58.420 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.127347.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.127347.log.gz 2026-02-19T21:12:58.420 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42397.log 2026-02-19T21:12:58.421 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75825.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75825.log.gz 2026-02-19T21:12:58.421 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19099.log 2026-02-19T21:12:58.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.42397.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65478.log 2026-02-19T21:12:58.422 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42397.log.gz 2026-02-19T21:12:58.422 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.19099.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19099.log.gz 2026-02-19T21:12:58.423 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79203.log 2026-02-19T21:12:58.423 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.65478.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65478.log.gz 2026-02-19T21:12:58.424 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59125.log 2026-02-19T21:12:58.424 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79203.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79203.log.gz 2026-02-19T21:12:58.424 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28579.log 2026-02-19T21:12:58.425 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59125.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59125.log.gz 2026-02-19T21:12:58.425 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18683.log 2026-02-19T21:12:58.426 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28579.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.28579.log.gz 2026-02-19T21:12:58.426 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31004.log 2026-02-19T21:12:58.427 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18683.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18683.log.gz 2026-02-19T21:12:58.427 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114914.log 2026-02-19T21:12:58.428 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.31004.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.31004.log.gz 2026-02-19T21:12:58.428 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125944.log 2026-02-19T21:12:58.428 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.114914.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114914.log.gz 2026-02-19T21:12:58.429 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25184.log 2026-02-19T21:12:58.429 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.125944.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125944.log.gz 2026-02-19T21:12:58.430 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100635.log 2026-02-19T21:12:58.430 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25184.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.25184.log.gz 2026-02-19T21:12:58.431 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56021.log 2026-02-19T21:12:58.431 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.100635.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100635.log.gz 2026-02-19T21:12:58.431 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24796.log 2026-02-19T21:12:58.432 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.56021.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56021.log.gz 2026-02-19T21:12:58.432 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68520.log 2026-02-19T21:12:58.433 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.24796.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24796.log.gz 2026-02-19T21:12:58.433 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94196.log 2026-02-19T21:12:58.434 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68520.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68520.log.gz 2026-02-19T21:12:58.434 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48244.log 2026-02-19T21:12:58.435 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.94196.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.94196.log.gz 2026-02-19T21:12:58.436 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30921.log 2026-02-19T21:12:58.436 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48244.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48244.log.gz 2026-02-19T21:12:58.436 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47371.log 2026-02-19T21:12:58.437 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30921.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127747.log 2026-02-19T21:12:58.442 INFO:teuthology.orchestra.run.trial151.stderr: 96.9% -- replaced with /var/log/ceph/cluster2-client.mirror.30921.log.gz 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72946.log.gz 2026-02-19T21:12:58.442 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:12:58.443 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47371.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47371.log.gz 2026-02-19T21:12:58.443 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99730.log 2026-02-19T21:12:58.444 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.127747.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19427.log 2026-02-19T21:12:58.444 INFO:teuthology.orchestra.run.trial151.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.127747.log.gz 2026-02-19T21:12:58.444 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.99730.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127868.log 2026-02-19T21:12:58.445 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.19427.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74007.log 2026-02-19T21:12:58.445 INFO:teuthology.orchestra.run.trial151.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.99730.log.gz 2026-02-19T21:12:58.448 INFO:teuthology.orchestra.run.trial151.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.19427.log.gz 2026-02-19T21:12:58.449 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127868.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123736.log 2026-02-19T21:12:58.449 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127868.log.gz 2026-02-19T21:12:58.450 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74007.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74007.log.gz 2026-02-19T21:12:58.450 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123848.log 2026-02-19T21:12:58.450 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123736.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123736.log.gz 2026-02-19T21:12:58.451 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55342.log 2026-02-19T21:12:58.451 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38416.log 2026-02-19T21:12:58.452 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123848.log: /var/log/ceph/cluster2-client.mirror.55342.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123848.log.gz 2026-02-19T21:12:58.452 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55342.log.gz 2026-02-19T21:12:58.452 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31643.log 2026-02-19T21:12:58.452 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105244.log 2026-02-19T21:12:58.453 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.38416.log: /var/log/ceph/cluster2-client.mirror.31643.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38416.log.gz 2026-02-19T21:12:58.453 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31643.log.gzgzip 2026-02-19T21:12:58.453 INFO:teuthology.orchestra.run.trial151.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48376.log 2026-02-19T21:12:58.453 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105244.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105244.log.gz 2026-02-19T21:12:58.454 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45181.log 2026-02-19T21:12:58.454 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59672.log 2026-02-19T21:12:58.455 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48376.log: /var/log/ceph/cluster2-client.mirror.45181.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.48376.log.gz 2026-02-19T21:12:58.455 INFO:teuthology.orchestra.run.trial151.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45181.log.gz 2026-02-19T21:12:58.455 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127374.log 2026-02-19T21:12:58.456 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79916.log 2026-02-19T21:12:58.456 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59672.log: /var/log/ceph/cluster1-client.mirror.127374.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59672.log.gz 2026-02-19T21:12:58.456 INFO:teuthology.orchestra.run.trial151.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.127374.log.gz 2026-02-19T21:12:58.456 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96251.log 2026-02-19T21:12:58.457 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88506.log 2026-02-19T21:12:58.457 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.79916.log: /var/log/ceph/cluster2-client.mirror.96251.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96251.log.gz 2026-02-19T21:12:58.457 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25025.log 2026-02-19T21:12:58.458 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.88506.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88506.log.gz 2026-02-19T21:12:58.464 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83094.log.gz 2026-02-19T21:12:58.473 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55827.log 2026-02-19T21:12:58.474 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35129.log 2026-02-19T21:12:58.474 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25025.log: /var/log/ceph/cluster2-client.mirror.55827.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25025.log.gz 2026-02-19T21:12:58.475 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55827.log.gz 2026-02-19T21:12:58.475 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93401.log 2026-02-19T21:12:58.475 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35129.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53484.log 2026-02-19T21:12:58.475 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.93401.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35129.log.gz 2026-02-19T21:12:58.475 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20044.log 2026-02-19T21:12:58.476 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53484.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53484.log.gz 2026-02-19T21:12:58.477 INFO:teuthology.orchestra.run.trial151.stderr: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.69715.log.gz 2026-02-19T21:12:58.489 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76862.log 2026-02-19T21:12:58.490 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.20044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96914.log 2026-02-19T21:12:58.490 INFO:teuthology.orchestra.run.trial151.stderr: 62.9% -- replaced with /var/log/ceph/cluster2-client.mirror.20044.log.gz 2026-02-19T21:12:58.490 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90519.log 2026-02-19T21:12:58.491 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96914.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96914.log.gz 2026-02-19T21:12:58.491 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73681.log 2026-02-19T21:12:58.491 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90519.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90519.log.gz 2026-02-19T21:12:58.492 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128291.log 2026-02-19T21:12:58.492 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.73681.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73681.log.gz 2026-02-19T21:12:58.493 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67224.log 2026-02-19T21:12:58.493 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.128291.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.128291.log.gz 2026-02-19T21:12:58.493 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51928.log 2026-02-19T21:12:58.494 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67224.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67224.log.gz 2026-02-19T21:12:58.494 INFO:teuthology.orchestra.run.trial151.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76862.log.gz 2026-02-19T21:12:58.494 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58993.log 2026-02-19T21:12:58.495 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.51928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72841.log 2026-02-19T21:12:58.495 INFO:teuthology.orchestra.run.trial151.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51928.log.gz 2026-02-19T21:12:58.495 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.58993.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69647.log 2026-02-19T21:12:58.495 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58993.log.gz 2026-02-19T21:12:58.505 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72841.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72152.log 2026-02-19T21:12:58.509 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.69647.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.69647.log.gz 2026-02-19T21:12:58.509 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68714.log 2026-02-19T21:12:58.510 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.72152.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72152.log.gz 2026-02-19T21:12:58.511 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.128087.log 2026-02-19T21:12:58.511 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68714.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68714.log.gz 2026-02-19T21:12:58.512 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.127429.log 2026-02-19T21:12:58.513 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.128087.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.128087.log.gz 2026-02-19T21:12:58.513 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107497.log 2026-02-19T21:12:58.515 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.2.127429.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.2.127429.log.gz 2026-02-19T21:12:58.516 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.106093.log 2026-02-19T21:12:58.517 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.107497.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.107497.log.gz 2026-02-19T21:12:58.518 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25863.log 2026-02-19T21:12:58.518 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.2.106093.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.85329.log.gz 2026-02-19T21:12:58.521 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30836.log 2026-02-19T21:12:58.523 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.25863.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25863.log.gz 2026-02-19T21:12:58.523 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85732.log 2026-02-19T21:12:58.523 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.30836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113755.log 2026-02-19T21:12:58.527 INFO:teuthology.orchestra.run.trial151.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.30836.log.gz 2026-02-19T21:12:58.528 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85732.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85732.log.gz 2026-02-19T21:12:58.528 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22844.log 2026-02-19T21:12:58.529 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113755.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113755.log.gz 2026-02-19T21:12:58.529 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77652.log 2026-02-19T21:12:58.530 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22844.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.22844.log.gz 2026-02-19T21:12:58.530 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31063.log 2026-02-19T21:12:58.531 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.77652.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77652.log.gz 2026-02-19T21:12:58.531 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126669.log 2026-02-19T21:12:58.532 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31063.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.65139.log 2026-02-19T21:12:58.541 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101517.log 2026-02-19T21:12:58.542 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126669.log: /var/log/ceph/cluster2-client.mirror.2.65139.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126669.log.gz 2026-02-19T21:12:58.557 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70406.log 2026-02-19T21:12:58.558 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.101517.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101517.log.gz 2026-02-19T21:12:58.561 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70018.log 2026-02-19T21:12:58.566 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70406.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70406.log.gz 2026-02-19T21:12:58.567 INFO:teuthology.orchestra.run.trial151.stderr: 92.8% 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.106093.log.gz 2026-02-19T21:12:58.567 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.2.65139.log.gz 2026-02-19T21:12:58.568 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118912.log 2026-02-19T21:12:58.569 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70018.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70018.log.gz 2026-02-19T21:12:58.569 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55113.log 2026-02-19T21:12:58.569 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118912.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.118912.log.gz 2026-02-19T21:12:58.569 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32675.log 2026-02-19T21:12:58.570 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.55113.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55113.log.gz 2026-02-19T21:12:58.570 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76130.log 2026-02-19T21:12:58.570 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32675.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74651.log 2026-02-19T21:12:58.571 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32675.log.gz 2026-02-19T21:12:58.571 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76130.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76130.log.gz 2026-02-19T21:12:58.571 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75333.log 2026-02-19T21:12:58.572 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.74651.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74651.log.gz 2026-02-19T21:12:58.572 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46248.log 2026-02-19T21:12:58.573 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75333.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75333.log.gz 2026-02-19T21:12:58.573 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46116.log 2026-02-19T21:12:58.574 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46248.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46248.log.gz 2026-02-19T21:12:58.574 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50867.log 2026-02-19T21:12:58.575 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.46116.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46116.log.gz 2026-02-19T21:12:58.575 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104367.log 2026-02-19T21:12:58.576 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50867.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50867.log.gz 2026-02-19T21:12:58.576 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109912.log 2026-02-19T21:12:58.577 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104367.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104367.log.gz 2026-02-19T21:12:58.577 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35711.log 2026-02-19T21:12:58.585 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.109912.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31546.log 2026-02-19T21:12:58.597 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35711.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.35711.log.gz 2026-02-19T21:12:58.601 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126047.log 2026-02-19T21:12:58.602 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.31546.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31546.log.gz 2026-02-19T21:12:58.606 INFO:teuthology.orchestra.run.trial151.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.82729.log.gz 2026-02-19T21:12:58.606 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108606.log 2026-02-19T21:12:58.607 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126047.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126047.log.gz 2026-02-19T21:12:58.607 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53096.log 2026-02-19T21:12:58.607 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108606.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108606.log.gz 2026-02-19T21:12:58.608 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67515.log 2026-02-19T21:12:58.608 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.53096.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53096.log.gz 2026-02-19T21:12:58.609 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50126.log 2026-02-19T21:12:58.609 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67515.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67515.log.gz 2026-02-19T21:12:58.609 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57344.log 2026-02-19T21:12:58.610 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50126.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50126.log.gz 2026-02-19T21:12:58.610 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67709.log 2026-02-19T21:12:58.611 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.57344.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57344.log.gz 2026-02-19T21:12:58.611 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54725.log 2026-02-19T21:12:58.612 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.67709.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67709.log.gz 2026-02-19T21:12:58.612 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123398.log 2026-02-19T21:12:58.613 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54725.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54725.log.gz 2026-02-19T21:12:58.613 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47177.log 2026-02-19T21:12:58.614 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123398.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123398.log.gz 2026-02-19T21:12:58.614 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85578.log 2026-02-19T21:12:58.615 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.47177.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47177.log.gz 2026-02-19T21:12:58.615 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63979.log 2026-02-19T21:12:58.615 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.85578.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85578.log.gz 2026-02-19T21:12:58.616 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.98858.log 2026-02-19T21:12:58.616 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.63979.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63979.log.gz 2026-02-19T21:12:58.617 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119196.log 2026-02-19T21:12:58.617 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69692.log.gz/var/log/ceph/cluster1-client.mirror.1.98858.log: 2026-02-19T21:12:58.617 INFO:teuthology.orchestra.run.trial151.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.79939.log.gz 2026-02-19T21:12:58.633 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48861.log 2026-02-19T21:12:58.634 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119196.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123196.log 2026-02-19T21:12:58.634 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.48861.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119196.log.gz 2026-02-19T21:12:58.634 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48861.log.gz 2026-02-19T21:12:58.634 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18872.log 2026-02-19T21:12:58.635 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.123196.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119074.log 2026-02-19T21:12:58.635 INFO:teuthology.orchestra.run.trial151.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.123196.log.gz 2026-02-19T21:12:58.635 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.18872.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18872.log.gz 2026-02-19T21:12:58.636 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80117.log.gz 2026-02-19T21:12:58.638 INFO:teuthology.orchestra.run.trial151.stderr: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.98858.log.gz 2026-02-19T21:12:58.638 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127586.log 2026-02-19T21:12:58.639 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119074.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32128.log 2026-02-19T21:12:58.639 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127586.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59513.log 2026-02-19T21:12:58.639 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32128.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127586.log.gz 2026-02-19T21:12:58.639 INFO:teuthology.orchestra.run.trial151.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.119074.log.gz 2026-02-19T21:12:58.639 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32128.log.gz 2026-02-19T21:12:58.640 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97637.log 2026-02-19T21:12:58.641 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.59513.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33154.log 2026-02-19T21:12:58.641 INFO:teuthology.orchestra.run.trial151.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59513.log.gz 2026-02-19T21:12:58.641 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.97637.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61651.log 2026-02-19T21:12:58.641 INFO:teuthology.orchestra.run.trial151.stderr: 88.4%/var/log/ceph/cluster2-client.mirror.33154.log: -- replaced with /var/log/ceph/cluster1-client.mirror.97637.log.gz 2026-02-19T21:12:58.641 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34741.log 2026-02-19T21:12:58.642 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.61651.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126688.log 2026-02-19T21:12:58.642 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33154.log.gz 2026-02-19T21:12:58.642 INFO:teuthology.orchestra.run.trial151.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61651.log.gz 2026-02-19T21:12:58.643 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.34741.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125637.log 2026-02-19T21:12:58.643 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.126688.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34741.log.gz 2026-02-19T21:12:58.643 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21542.log 2026-02-19T21:12:58.643 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.125637.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.126688.log.gz 2026-02-19T21:12:58.643 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71764.log 2026-02-19T21:12:58.643 INFO:teuthology.orchestra.run.trial151.stderr: 86.3% -- replaced with /var/log/ceph/cluster1-client.mirror.125637.log.gz 2026-02-19T21:12:58.643 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.21542.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108044.log 2026-02-19T21:12:58.644 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.71764.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21542.log.gz 2026-02-19T21:12:58.644 INFO:teuthology.orchestra.run.trial151.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71764.log.gz 2026-02-19T21:12:58.644 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40360.log 2026-02-19T21:12:58.644 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90910.log 2026-02-19T21:12:58.644 INFO:teuthology.orchestra.run.trial151.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.40360.log: -- replaced with /var/log/ceph/cluster2-client.mirror.108044.log.gz 2026-02-19T21:12:58.644 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90225.log 2026-02-19T21:12:58.645 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90910.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128788.log 2026-02-19T21:12:58.645 INFO:teuthology.orchestra.run.trial151.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40360.log.gz 2026-02-19T21:12:58.645 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90225.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90910.log.gz 2026-02-19T21:12:58.645 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19805.log 2026-02-19T21:12:58.645 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.admin.128788.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22565.log 2026-02-19T21:12:58.645 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128788.log.gz 2026-02-19T21:12:58.646 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.19805.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.19805.log.gz 2026-02-19T21:12:58.661 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28191.log 2026-02-19T21:12:58.662 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.22565.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44304.log 2026-02-19T21:12:58.662 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.28191.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.22565.log.gz 2026-02-19T21:12:58.663 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76246.log 2026-02-19T21:12:58.663 INFO:teuthology.orchestra.run.trial151.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.28191.log.gz 2026-02-19T21:12:58.663 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35773.log 2026-02-19T21:12:58.663 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.76246.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44304.log.gz 2026-02-19T21:12:58.663 INFO:teuthology.orchestra.run.trial151.stderr: 89.3%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117560.log 2026-02-19T21:12:58.664 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.76246.log.gz 2026-02-19T21:12:58.664 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35773.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127020.log 2026-02-19T21:12:58.664 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.117560.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35773.log.gz 2026-02-19T21:12:58.664 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36549.log 2026-02-19T21:12:58.664 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.127020.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105671.log 2026-02-19T21:12:58.665 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.36549.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.117560.log.gz 2026-02-19T21:12:58.665 INFO:teuthology.orchestra.run.trial151.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36549.log.gz 2026-02-19T21:12:58.665 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51202.log 2026-02-19T21:12:58.666 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.105671.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108370.log 2026-02-19T21:12:58.666 INFO:teuthology.orchestra.run.trial151.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105671.log.gz/var/log/ceph/cluster1-client.admin.51202.log: 2026-02-19T21:12:58.667 INFO:teuthology.orchestra.run.trial151.stderr: 88.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90823.log 2026-02-19T21:12:58.667 INFO:teuthology.orchestra.run.trial151.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.51202.log.gz 2026-02-19T21:12:58.667 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.108370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26251.log 2026-02-19T21:12:58.667 INFO:teuthology.orchestra.run.trial151.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.108370.log.gz 2026-02-19T21:12:58.667 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90823.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50514.log 2026-02-19T21:12:58.668 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.26251.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26251.log.gz 2026-02-19T21:12:58.668 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90724.log 2026-02-19T21:12:58.669 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50514.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50514.log.gz 2026-02-19T21:12:58.669 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96816.log 2026-02-19T21:12:58.670 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90724.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90724.log.gz 2026-02-19T21:12:58.670 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19226.log 2026-02-19T21:12:58.671 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.96816.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96816.log.gz 2026-02-19T21:12:58.671 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52184.log 2026-02-19T21:12:58.671 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.admin.19226.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19226.log.gz 2026-02-19T21:12:58.672 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44630.log 2026-02-19T21:12:58.672 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.52184.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52184.log.gz 2026-02-19T21:12:58.672 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70635.log 2026-02-19T21:12:58.673 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.44630.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44630.log.gz 2026-02-19T21:12:58.673 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66483.log 2026-02-19T21:12:58.674 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70635.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70635.log.gz 2026-02-19T21:12:58.674 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69245.log 2026-02-19T21:12:58.675 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.66483.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66483.log.gz 2026-02-19T21:12:58.675 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.42544.log 2026-02-19T21:12:58.675 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69245.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69245.log.gz 2026-02-19T21:12:58.676 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54919.log 2026-02-19T21:12:58.676 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.0.42544.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118932.log 2026-02-19T21:12:58.685 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78856.log 2026-02-19T21:12:58.686 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.118932.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.118932.log.gz 2026-02-19T21:12:58.689 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.54919.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54919.log.gz 2026-02-19T21:12:58.690 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97262.log 2026-02-19T21:12:58.690 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.78856.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78856.log.gz 2026-02-19T21:12:58.701 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69715.log 2026-02-19T21:12:58.702 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.97262.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97262.log.gz 2026-02-19T21:12:58.705 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90225.log.gz 2026-02-19T21:12:58.709 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90368.log 2026-02-19T21:12:58.725 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69715.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75922.log 2026-02-19T21:12:58.725 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.90368.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90368.log.gz 2026-02-19T21:12:58.726 INFO:teuthology.orchestra.run.trial151.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69715.log.gz 2026-02-19T21:12:58.726 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98043.log 2026-02-19T21:12:58.727 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.75922.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75922.log.gz 2026-02-19T21:12:58.727 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119520.log 2026-02-19T21:12:58.728 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.98043.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98043.log.gz 2026-02-19T21:12:58.728 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113064.log 2026-02-19T21:12:58.730 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.119520.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.119520.log.gz 2026-02-19T21:12:58.730 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68970.log 2026-02-19T21:12:58.731 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.113064.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.113064.log.gz 2026-02-19T21:12:58.731 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35226.log 2026-02-19T21:12:58.732 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.68970.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68970.log.gz 2026-02-19T21:12:58.732 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32810.log 2026-02-19T21:12:58.733 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.35226.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35226.log.gz 2026-02-19T21:12:58.733 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104637.log 2026-02-19T21:12:58.734 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.32810.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32810.log.gz 2026-02-19T21:12:58.734 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.81614.log 2026-02-19T21:12:58.735 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104637.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.104637.log.gz 2026-02-19T21:12:58.736 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50091.log 2026-02-19T21:12:58.744 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster1-client.mirror.0.81614.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78098.log.gz 2026-02-19T21:12:58.745 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102052.log 2026-02-19T21:12:58.746 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.50091.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50091.log.gz 2026-02-19T21:12:58.757 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69361.log 2026-02-19T21:12:58.758 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.102052.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.102052.log.gz 2026-02-19T21:12:58.758 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70503.log 2026-02-19T21:12:58.759 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.69361.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69361.log.gz 2026-02-19T21:12:58.759 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104173.log 2026-02-19T21:12:58.760 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.70503.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70503.log.gz 2026-02-19T21:12:58.760 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117918.log 2026-02-19T21:12:58.761 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.104173.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104173.log.gz 2026-02-19T21:12:58.762 INFO:teuthology.orchestra.run.trial151.stderr:/var/log/ceph/cluster2-client.mirror.117918.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.117918.log.gz 2026-02-19T21:12:58.770 INFO:teuthology.orchestra.run.trial151.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.0.42544.log.gz 2026-02-19T21:12:58.777 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80567.log.gz 2026-02-19T21:12:58.822 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.109912.log.gz 2026-02-19T21:12:58.872 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125318.log.gz 2026-02-19T21:12:58.944 INFO:teuthology.orchestra.run.trial151.stderr: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.38094.log.gz 2026-02-19T21:12:58.982 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78357.log.gz 2026-02-19T21:12:58.987 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79916.log.gz 2026-02-19T21:12:59.019 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81865.log.gz 2026-02-19T21:12:59.026 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90823.log.gz 2026-02-19T21:12:59.029 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72841.log.gz 2026-02-19T21:12:59.075 INFO:teuthology.orchestra.run.trial151.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.127020.log.gz 2026-02-19T21:12:59.123 INFO:teuthology.orchestra.run.trial151.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77829.log.gz 2026-02-19T21:12:59.134 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81082.log.gz 2026-02-19T21:12:59.134 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.93401.log.gz 2026-02-19T21:12:59.248 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31063.log.gz 2026-02-19T21:12:59.252 INFO:teuthology.orchestra.run.trial151.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54070.log.gz 2026-02-19T21:13:00.137 INFO:teuthology.orchestra.run.trial151.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.75649.log.gz 2026-02-19T21:13:02.523 INFO:teuthology.orchestra.run.trial151.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.3.79939.log.gz 2026-02-19T21:13:03.563 INFO:teuthology.orchestra.run.trial151.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.81614.log.gz 2026-02-19T21:13:07.597 INFO:teuthology.orchestra.run.trial151.stderr: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.1.19443.log.gz 2026-02-19T21:14:55.144 INFO:teuthology.orchestra.run.trial013.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-19T21:15:05.222 INFO:teuthology.orchestra.run.trial013.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-19T21:15:15.244 INFO:teuthology.orchestra.run.trial151.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-19T21:15:21.148 INFO:teuthology.orchestra.run.trial151.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-19T21:15:40.856 INFO:teuthology.orchestra.run.trial013.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-19T21:15:40.857 INFO:teuthology.orchestra.run.trial013.stderr: 2026-02-19T21:15:40.857 INFO:teuthology.orchestra.run.trial013.stderr:real 2m46.558s 2026-02-19T21:15:40.857 INFO:teuthology.orchestra.run.trial013.stderr:user 6m39.883s 2026-02-19T21:15:40.857 INFO:teuthology.orchestra.run.trial013.stderr:sys 0m19.325s 2026-02-19T21:16:05.182 INFO:teuthology.orchestra.run.trial151.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-19T21:16:05.183 INFO:teuthology.orchestra.run.trial151.stderr: 2026-02-19T21:16:05.183 INFO:teuthology.orchestra.run.trial151.stderr:real 3m10.875s 2026-02-19T21:16:05.183 INFO:teuthology.orchestra.run.trial151.stderr:user 9m19.672s 2026-02-19T21:16:05.183 INFO:teuthology.orchestra.run.trial151.stderr:sys 0m31.687s 2026-02-19T21:16:05.184 INFO:tasks.ceph:Archiving logs... 2026-02-19T21:16:05.185 DEBUG:teuthology.misc:Transferring archived files from trial013:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645/remote/trial013/log 2026-02-19T21:16:05.186 DEBUG:teuthology.orchestra.run.trial013:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-19T21:16:34.096 DEBUG:teuthology.misc:Transferring archived files from trial151:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645/remote/trial151/log 2026-02-19T21:16:34.101 DEBUG:teuthology.orchestra.run.trial151:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-19T21:17:13.059 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-19T21:17:13.073 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-19T21:17:13.073 DEBUG:teuthology.orchestra.run.trial013:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-19T21:17:13.085 DEBUG:teuthology.orchestra.run.trial151:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-19T21:17:13.205 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-19T21:17:13.217 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-19T21:17:13.217 DEBUG:teuthology.orchestra.run.trial013:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-19T21:17:13.219 DEBUG:teuthology.orchestra.run.trial151:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-19T21:17:13.267 INFO:teuthology.orchestra.run.trial013.stdout: remote refid st t when poll reach delay offset jitter 2026-02-19T21:17:13.267 INFO:teuthology.orchestra.run.trial013.stdout:============================================================================== 2026-02-19T21:17:13.267 INFO:teuthology.orchestra.run.trial013.stdout:+ntp.maxhost.io 132.163.97.4 2 u 42 128 377 69.669 +0.566 3.462 2026-02-19T21:17:13.267 INFO:teuthology.orchestra.run.trial013.stdout:*ntp.maxhost.io 132.163.97.4 2 u 102 128 377 69.720 +0.655 3.835 2026-02-19T21:17:13.267 INFO:teuthology.orchestra.run.trial013.stdout:+23.155.72.147 ( 158.51.99.19 3 u 23 128 377 29.952 +1.617 0.284 2026-02-19T21:17:13.267 INFO:teuthology.orchestra.run.trial013.stdout: 104.233.211.205 .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2026-02-19T21:17:13.267 INFO:teuthology.orchestra.run.trial151.stdout: remote refid st t when poll reach delay offset jitter 2026-02-19T21:17:13.268 INFO:teuthology.orchestra.run.trial151.stdout:============================================================================== 2026-02-19T21:17:13.268 INFO:teuthology.orchestra.run.trial151.stdout:+ntp.maxhost.io 132.163.97.4 2 u 30 64 377 69.661 +1.565 5.070 2026-02-19T21:17:13.268 INFO:teuthology.orchestra.run.trial151.stdout:*ntp.maxhost.io 132.163.97.4 2 u 27 64 337 69.721 +1.568 0.508 2026-02-19T21:17:13.268 INFO:teuthology.orchestra.run.trial151.stdout:+23.155.72.147 ( 158.51.99.19 3 u 20 64 377 30.086 +2.010 0.193 2026-02-19T21:17:13.268 INFO:teuthology.orchestra.run.trial151.stdout: 104.233.211.205 .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2026-02-19T21:17:13.268 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-19T21:17:13.279 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-19T21:17:13.279 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-19T21:17:13.289 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-19T21:17:13.302 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-19T21:17:13.315 INFO:teuthology.task.internal:Duration was 3133.870413 seconds 2026-02-19T21:17:13.315 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-19T21:17:13.328 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-19T21:17:13.328 DEBUG:teuthology.orchestra.run.trial013:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-19T21:17:13.329 DEBUG:teuthology.orchestra.run.trial151:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-19T21:17:13.353 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-19T21:17:13.353 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial013.front.sepia.ceph.com 2026-02-19T21:17:13.353 DEBUG:teuthology.orchestra.run.trial013:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-19T21:17:13.399 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial151.front.sepia.ceph.com 2026-02-19T21:17:13.399 DEBUG:teuthology.orchestra.run.trial151:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-19T21:17:13.407 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-19T21:17:13.407 DEBUG:teuthology.orchestra.run.trial013:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-19T21:17:13.442 DEBUG:teuthology.orchestra.run.trial151:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-19T21:17:13.680 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-19T21:17:13.680 DEBUG:teuthology.orchestra.run.trial013:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-19T21:17:13.683 DEBUG:teuthology.orchestra.run.trial151:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-19T21:17:13.688 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-19T21:17:13.688 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-19T21:17:13.689 INFO:teuthology.orchestra.run.trial013.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-19T21:17:13.689 INFO:teuthology.orchestra.run.trial013.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-19T21:17:13.689 INFO:teuthology.orchestra.run.trial013.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: /home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-19T21:17:13.692 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-19T21:17:13.693 INFO:teuthology.orchestra.run.trial151.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-19T21:17:13.693 INFO:teuthology.orchestra.run.trial151.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-19T21:17:13.693 INFO:teuthology.orchestra.run.trial151.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-19T21:17:13.694 INFO:teuthology.orchestra.run.trial151.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: /home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-19T21:17:13.755 INFO:teuthology.orchestra.run.trial013.stderr: 81.4% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-19T21:17:13.762 INFO:teuthology.orchestra.run.trial151.stderr: 81.5% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-19T21:17:13.764 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-19T21:17:13.783 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-19T21:17:13.784 DEBUG:teuthology.orchestra.run.trial013:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-19T21:17:13.809 DEBUG:teuthology.orchestra.run.trial151:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-19T21:17:13.818 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-19T21:17:13.839 DEBUG:teuthology.orchestra.run.trial013:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-19T21:17:13.854 DEBUG:teuthology.orchestra.run.trial151:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-19T21:17:13.860 INFO:teuthology.orchestra.run.trial013.stdout:kernel.core_pattern = core 2026-02-19T21:17:13.868 INFO:teuthology.orchestra.run.trial151.stdout:kernel.core_pattern = core 2026-02-19T21:17:13.877 DEBUG:teuthology.orchestra.run.trial013:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-19T21:17:13.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T21:17:13.913 DEBUG:teuthology.orchestra.run.trial151:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-19T21:17:13.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-19T21:17:13.924 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-19T21:17:13.943 INFO:teuthology.task.internal:Transferring archived files... 2026-02-19T21:17:13.945 DEBUG:teuthology.misc:Transferring archived files from trial013:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645/remote/trial013 2026-02-19T21:17:13.945 DEBUG:teuthology.orchestra.run.trial013:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-19T21:17:13.985 DEBUG:teuthology.misc:Transferring archived files from trial151:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-19_15:47:56-rbd-wip-yuri-testing-2026-02-18-1820-distro-default-trial/59645/remote/trial151 2026-02-19T21:17:13.985 DEBUG:teuthology.orchestra.run.trial151:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-19T21:17:14.006 INFO:teuthology.task.internal:Removing archive directory... 2026-02-19T21:17:14.006 DEBUG:teuthology.orchestra.run.trial013:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-19T21:17:14.018 DEBUG:teuthology.orchestra.run.trial151:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-19T21:17:14.044 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-19T21:17:14.056 INFO:teuthology.task.internal:Not uploading archives. 2026-02-19T21:17:14.056 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-19T21:17:14.075 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-19T21:17:14.075 DEBUG:teuthology.orchestra.run.trial013:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-19T21:17:14.077 DEBUG:teuthology.orchestra.run.trial151:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-19T21:17:14.080 INFO:teuthology.orchestra.run.trial013.stdout: 1179687 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 19 21:17 /home/ubuntu/cephtest 2026-02-19T21:17:14.089 INFO:teuthology.orchestra.run.trial151.stdout: 1179687 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 19 21:17 /home/ubuntu/cephtest 2026-02-19T21:17:14.090 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-19T21:17:14.110 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-19T21:17:14.147 INFO:teuthology.run:Summary data: description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit} duration: 3133.8704133033752 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-19T21:17:14.147 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-19T21:17:14.248 INFO:teuthology.run:pass